From 867ba026b7ae2c0430182d74287ab297db4dc963 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Thu, 18 Jul 2024 17:19:49 +0100 Subject: [PATCH] Finish docker-compose files --- Makefile | 2 +- docker-compose-azure.yaml | 50 ++++++++++++++++++++++++++++-------- docker-compose-claude.yaml | 50 ++++++++++++++++++++++++++++-------- docker-compose-ollama.yaml | 25 +++++++++--------- docker-compose-vertexai.yaml | 50 ++++++++++++++++++++++++++++-------- grafana/dashboard.json | 14 +++++----- grafana/dashboard.yml | 10 +++++--- grafana/datasource.yml | 18 ++++++++++--- setup.py | 2 +- 9 files changed, 163 insertions(+), 58 deletions(-) diff --git a/Makefile b/Makefile index bc4e7340..8fb5e284 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # VERSION=$(shell git describe | sed 's/^v//') -VERSION=0.4.0 +VERSION=0.4.1 all: container diff --git a/docker-compose-azure.yaml b/docker-compose-azure.yaml index b5c63d64..d129f384 100644 --- a/docker-compose-azure.yaml +++ b/docker-compose-azure.yaml @@ -6,6 +6,8 @@ volumes: etcd: minio-data: milvus: + prometheus-data: + grafana-storage: services: @@ -90,8 +92,34 @@ services: - "milvus:/var/lib/milvus" restart: on-failure:100 + prometheus: + image: docker.io/prom/prometheus:v2.53.1 + ports: + - "9090:9090" + volumes: + - "./prometheus:/etc/prometheus" + - "prometheus-data:/prometheus" + restart: on-failure:100 + + grafana: + image: docker.io/grafana/grafana:10.0.0 + ports: + - "3000:3000" + volumes: + - "grafana-storage:/var/lib/grafana" + - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml" + - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml" + - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json" + environment: +# GF_AUTH_ANONYMOUS_ORG_ROLE: Admin +# GF_AUTH_ANONYMOUS_ENABLED: true +# GF_ORG_ROLE: Admin + GF_ORG_NAME: trustgraph.ai +# GF_SERVER_ROOT_URL: https://example.com + restart: on-failure:100 + pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "pdf-decoder" - "-p" @@ -99,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "chunker-recursive" - "-p" @@ -107,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "embeddings-vectorize" - "-p" @@ -115,15 +143,17 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" + - "-m" + - "mixedbread-ai/mxbai-embed-large-v1" restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "kg-extract-definitions" - "-p" @@ -131,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "kg-extract-relationships" - "-p" @@ -139,7 +169,7 @@ services: restart: on-failure:100 vector-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "vector-write-milvus" - "-p" @@ -149,7 +179,7 @@ services: restart: on-failure:100 graph-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "graph-write-cassandra" - "-p" @@ -159,7 +189,7 @@ services: restart: on-failure:100 llm: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "llm-azure-text" - "-p" @@ -171,7 +201,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "graph-rag" - "-p" diff --git a/docker-compose-claude.yaml b/docker-compose-claude.yaml index 7ade6a7f..40a3809a 100644 --- a/docker-compose-claude.yaml +++ b/docker-compose-claude.yaml @@ -6,6 +6,8 @@ volumes: etcd: minio-data: milvus: + prometheus-data: + grafana-storage: services: @@ -90,8 +92,34 @@ services: - "milvus:/var/lib/milvus" restart: on-failure:100 + prometheus: + image: docker.io/prom/prometheus:v2.53.1 + ports: + - "9090:9090" + volumes: + - "./prometheus:/etc/prometheus" + - "prometheus-data:/prometheus" + restart: on-failure:100 + + grafana: + image: docker.io/grafana/grafana:10.0.0 + ports: + - "3000:3000" + volumes: + - "grafana-storage:/var/lib/grafana" + - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml" + - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml" + - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json" + environment: +# GF_AUTH_ANONYMOUS_ORG_ROLE: Admin +# GF_AUTH_ANONYMOUS_ENABLED: true +# GF_ORG_ROLE: Admin + GF_ORG_NAME: trustgraph.ai +# GF_SERVER_ROOT_URL: https://example.com + restart: on-failure:100 + pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "pdf-decoder" - "-p" @@ -99,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "chunker-recursive" - "-p" @@ -107,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "embeddings-vectorize" - "-p" @@ -115,15 +143,17 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" + - "-m" + - "mixedbread-ai/mxbai-embed-large-v1" restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "kg-extract-definitions" - "-p" @@ -131,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "kg-extract-relationships" - "-p" @@ -139,7 +169,7 @@ services: restart: on-failure:100 vector-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "vector-write-milvus" - "-p" @@ -149,7 +179,7 @@ services: restart: on-failure:100 graph-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "graph-write-cassandra" - "-p" @@ -159,7 +189,7 @@ services: restart: on-failure:100 llm: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "llm-claude-text" - "-p" @@ -169,7 +199,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "graph-rag" - "-p" diff --git a/docker-compose-ollama.yaml b/docker-compose-ollama.yaml index 0f61a6b9..6188e0df 100644 --- a/docker-compose-ollama.yaml +++ b/docker-compose-ollama.yaml @@ -111,14 +111,15 @@ services: - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml" - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json" environment: - GF_AUTH_ANONYMOUS_ORG_ROLE: Admin - GF_AUTH_ANONYMOUS_ENABLED: true +# GF_AUTH_ANONYMOUS_ORG_ROLE: Admin +# GF_AUTH_ANONYMOUS_ENABLED: true +# GF_ORG_ROLE: Admin GF_ORG_NAME: trustgraph.ai # GF_SERVER_ROOT_URL: https://example.com restart: on-failure:100 pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "pdf-decoder" - "-p" @@ -126,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "chunker-recursive" - "-p" @@ -134,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "embeddings-vectorize" - "-p" @@ -142,7 +143,7 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "embeddings-hf" - "-p" @@ -152,7 +153,7 @@ services: restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "kg-extract-definitions" - "-p" @@ -160,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "kg-extract-relationships" - "-p" @@ -168,7 +169,7 @@ services: restart: on-failure:100 vector-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "vector-write-milvus" - "-p" @@ -178,7 +179,7 @@ services: restart: on-failure:100 graph-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "graph-write-cassandra" - "-p" @@ -188,7 +189,7 @@ services: restart: on-failure:100 llm: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "llm-ollama-text" - "-p" @@ -198,7 +199,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "graph-rag" - "-p" diff --git a/docker-compose-vertexai.yaml b/docker-compose-vertexai.yaml index 5cae73a9..0043efe0 100644 --- a/docker-compose-vertexai.yaml +++ b/docker-compose-vertexai.yaml @@ -6,6 +6,8 @@ volumes: etcd: minio-data: milvus: + prometheus-data: + grafana-storage: services: @@ -90,8 +92,34 @@ services: - "milvus:/var/lib/milvus" restart: on-failure:100 + prometheus: + image: docker.io/prom/prometheus:v2.53.1 + ports: + - "9090:9090" + volumes: + - "./prometheus:/etc/prometheus" + - "prometheus-data:/prometheus" + restart: on-failure:100 + + grafana: + image: docker.io/grafana/grafana:10.0.0 + ports: + - "3000:3000" + volumes: + - "grafana-storage:/var/lib/grafana" + - "./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml" + - "./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml" + - "./grafana/dashboard.json:/var/lib/grafana/dashboards/dashboard.json" + environment: +# GF_AUTH_ANONYMOUS_ORG_ROLE: Admin +# GF_AUTH_ANONYMOUS_ENABLED: true +# GF_ORG_ROLE: Admin + GF_ORG_NAME: trustgraph.ai +# GF_SERVER_ROOT_URL: https://example.com + restart: on-failure:100 + pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "pdf-decoder" - "-p" @@ -99,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "chunker-recursive" - "-p" @@ -107,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "embeddings-vectorize" - "-p" @@ -115,15 +143,17 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "embeddings-hf" - "-p" - "pulsar://pulsar:6650" + - "-m" + - "mixedbread-ai/mxbai-embed-large-v1" restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "kg-extract-definitions" - "-p" @@ -131,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "kg-extract-relationships" - "-p" @@ -139,7 +169,7 @@ services: restart: on-failure:100 vector-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "vector-write-milvus" - "-p" @@ -149,7 +179,7 @@ services: restart: on-failure:100 graph-write: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "graph-write-cassandra" - "-p" @@ -159,7 +189,7 @@ services: restart: on-failure:100 llm: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "llm-vertexai-text" - "-p" @@ -173,7 +203,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.4.0 + image: docker.io/trustgraph/trustgraph-flow:0.4.1 command: - "graph-rag" - "-p" diff --git a/grafana/dashboard.json b/grafana/dashboard.json index 4535a3bd..8ed62fe6 100644 --- a/grafana/dashboard.json +++ b/grafana/dashboard.json @@ -25,7 +25,7 @@ { "datasource": { "type": "prometheus", - "uid": "b7a520e5-9efe-4e0b-961b-9d1d3854800d" + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" }, "fieldConfig": { "defaults": { @@ -79,7 +79,7 @@ { "datasource": { "type": "prometheus", - "uid": "b7a520e5-9efe-4e0b-961b-9d1d3854800d" + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" }, "editorMode": "builder", "expr": "avg(rate(request_latency_bucket{instance=\"llm:8000\"}[5m]))", @@ -94,7 +94,7 @@ { "datasource": { "type": "prometheus", - "uid": "b7a520e5-9efe-4e0b-961b-9d1d3854800d" + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" }, "fieldConfig": { "defaults": { @@ -171,7 +171,7 @@ { "datasource": { "type": "prometheus", - "uid": "b7a520e5-9efe-4e0b-961b-9d1d3854800d" + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" }, "editorMode": "builder", "expr": "sum by(status) (rate(processing_count_total[5m]))", @@ -188,7 +188,7 @@ { "datasource": { "type": "prometheus", - "uid": "b7a520e5-9efe-4e0b-961b-9d1d3854800d" + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" }, "fieldConfig": { "defaults": { @@ -265,7 +265,7 @@ { "datasource": { "type": "prometheus", - "uid": "b7a520e5-9efe-4e0b-961b-9d1d3854800d" + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" }, "editorMode": "builder", "expr": "rate(request_latency_count[1m])", @@ -291,7 +291,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Trustgraph", + "title": "Overview", "uid": "b5c8abf8-fe79-496b-b028-10bde917d1f0", "version": 7, "weekStart": "" diff --git a/grafana/dashboard.yml b/grafana/dashboard.yml index ee4891d1..9b9e7450 100644 --- a/grafana/dashboard.yml +++ b/grafana/dashboard.yml @@ -1,13 +1,17 @@ + apiVersion: 1 providers: - - name: 'Trustgraph' + - name: 'trustgraph.ai' + orgId: 1 + folder: 'TrustGraph' + folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228' type: file disableDeletion: false - editable: true updateIntervalSeconds: 30 - allowUiUpdates: false + allowUiUpdates: true options: path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/grafana/datasource.yml b/grafana/datasource.yml index 9e4c8108..3afdb9b7 100644 --- a/grafana/datasource.yml +++ b/grafana/datasource.yml @@ -1,11 +1,21 @@ apiVersion: 1 +prune: true + datasources: - name: Prometheus type: prometheus access: proxy - url: http://prometheus:9090 - basicAuth: false - isDefault: true - readOnly: false + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129' + + url: http://prometheus:9090 + + basicAuth: false + withCredentials: false + isDefault: true + editable: true diff --git a/setup.py b/setup.py index a3ae5e2b..b1c83fc4 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import os with open("README.md", "r") as fh: long_description = fh.read() -version = "0.4.0" +version = "0.4.1" setuptools.setup( name="trustgraph",