diff --git a/deploy/docker-compose/tg-launch-azure-cassandra.yaml b/deploy/docker-compose/tg-launch-azure-cassandra.yaml index f7599d12..1a681eb5 100644 --- a/deploy/docker-compose/tg-launch-azure-cassandra.yaml +++ b/deploy/docker-compose/tg-launch-azure-cassandra.yaml @@ -33,7 +33,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -50,7 +50,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -95,7 +95,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -130,7 +130,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -145,7 +145,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pdf-decoder: command: @@ -160,7 +175,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -262,6 +277,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -339,7 +365,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -429,6 +455,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -506,7 +543,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -560,7 +597,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -577,7 +614,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -594,7 +631,7 @@ services: reservations: cpus: '0.1' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -611,7 +648,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -628,7 +665,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -645,7 +682,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -668,7 +705,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -695,7 +732,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -710,7 +747,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: cassandra: {} diff --git a/deploy/docker-compose/tg-launch-azure-neo4j.yaml b/deploy/docker-compose/tg-launch-azure-neo4j.yaml index f6642ca0..2a57eb58 100644 --- a/deploy/docker-compose/tg-launch-azure-neo4j.yaml +++ b/deploy/docker-compose/tg-launch-azure-neo4j.yaml @@ -16,7 +16,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -33,7 +33,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -78,7 +78,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -113,7 +113,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -128,7 +128,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 neo4j: deploy: @@ -161,7 +176,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -263,6 +278,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -340,7 +366,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -430,6 +456,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -507,7 +544,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -561,7 +598,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -578,7 +615,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -595,7 +632,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -612,7 +649,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -629,7 +666,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -646,7 +683,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -669,7 +706,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -696,7 +733,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -711,7 +748,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: grafana-storage: {} diff --git a/deploy/docker-compose/tg-launch-bedrock-cassandra.yaml b/deploy/docker-compose/tg-launch-bedrock-cassandra.yaml index e8bca8b3..2eb3692a 100644 --- a/deploy/docker-compose/tg-launch-bedrock-cassandra.yaml +++ b/deploy/docker-compose/tg-launch-bedrock-cassandra.yaml @@ -33,7 +33,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -50,7 +50,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -95,7 +95,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -130,7 +130,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -145,7 +145,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pdf-decoder: command: @@ -160,7 +175,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -262,6 +277,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -339,7 +365,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -429,6 +455,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -506,7 +543,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -560,7 +597,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -577,7 +614,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -594,7 +631,7 @@ services: reservations: cpus: '0.1' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -611,7 +648,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -628,7 +665,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -645,7 +682,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -672,7 +709,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -703,7 +740,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -718,7 +755,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: cassandra: {} diff --git a/deploy/docker-compose/tg-launch-bedrock-neo4j.yaml b/deploy/docker-compose/tg-launch-bedrock-neo4j.yaml index c4fc9266..6c0d3a1d 100644 --- a/deploy/docker-compose/tg-launch-bedrock-neo4j.yaml +++ b/deploy/docker-compose/tg-launch-bedrock-neo4j.yaml @@ -16,7 +16,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -33,7 +33,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -78,7 +78,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -113,7 +113,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -128,7 +128,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 neo4j: deploy: @@ -161,7 +176,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -263,6 +278,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -340,7 +366,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -430,6 +456,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -507,7 +544,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -561,7 +598,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -578,7 +615,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -595,7 +632,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -612,7 +649,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -629,7 +666,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -646,7 +683,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -673,7 +710,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -704,7 +741,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -719,7 +756,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: grafana-storage: {} diff --git a/deploy/docker-compose/tg-launch-claude-cassandra.yaml b/deploy/docker-compose/tg-launch-claude-cassandra.yaml index 59db820d..af114880 100644 --- a/deploy/docker-compose/tg-launch-claude-cassandra.yaml +++ b/deploy/docker-compose/tg-launch-claude-cassandra.yaml @@ -33,7 +33,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -50,7 +50,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -95,7 +95,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -130,7 +130,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -145,7 +145,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pdf-decoder: command: @@ -160,7 +175,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -262,6 +277,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -339,7 +365,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -429,6 +455,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -506,7 +543,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -560,7 +597,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -577,7 +614,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -594,7 +631,7 @@ services: reservations: cpus: '0.1' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -611,7 +648,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -628,7 +665,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -645,7 +682,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -666,7 +703,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -691,7 +728,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -706,7 +743,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: cassandra: {} diff --git a/deploy/docker-compose/tg-launch-claude-neo4j.yaml b/deploy/docker-compose/tg-launch-claude-neo4j.yaml index 2d157d12..eefb4cee 100644 --- a/deploy/docker-compose/tg-launch-claude-neo4j.yaml +++ b/deploy/docker-compose/tg-launch-claude-neo4j.yaml @@ -16,7 +16,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -33,7 +33,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -78,7 +78,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -113,7 +113,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -128,7 +128,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 neo4j: deploy: @@ -161,7 +176,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -263,6 +278,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -340,7 +366,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -430,6 +456,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -507,7 +544,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -561,7 +598,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -578,7 +615,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -595,7 +632,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -612,7 +649,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -629,7 +666,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -646,7 +683,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -667,7 +704,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -692,7 +729,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -707,7 +744,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: grafana-storage: {} diff --git a/deploy/docker-compose/tg-launch-cohere-cassandra.yaml b/deploy/docker-compose/tg-launch-cohere-cassandra.yaml index f6497469..d0df4d9f 100644 --- a/deploy/docker-compose/tg-launch-cohere-cassandra.yaml +++ b/deploy/docker-compose/tg-launch-cohere-cassandra.yaml @@ -33,7 +33,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -50,7 +50,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -95,7 +95,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -130,7 +130,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -145,7 +145,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pdf-decoder: command: @@ -160,7 +175,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -262,6 +277,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -339,7 +365,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -429,6 +455,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -506,7 +543,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -560,7 +597,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -577,7 +614,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -594,7 +631,7 @@ services: reservations: cpus: '0.1' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -611,7 +648,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -628,7 +665,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -645,7 +682,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -664,7 +701,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -687,7 +724,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -702,7 +739,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: cassandra: {} diff --git a/deploy/docker-compose/tg-launch-cohere-neo4j.yaml b/deploy/docker-compose/tg-launch-cohere-neo4j.yaml index 048ad2a3..54ced668 100644 --- a/deploy/docker-compose/tg-launch-cohere-neo4j.yaml +++ b/deploy/docker-compose/tg-launch-cohere-neo4j.yaml @@ -16,7 +16,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -33,7 +33,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -78,7 +78,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -113,7 +113,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -128,7 +128,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 neo4j: deploy: @@ -161,7 +176,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -263,6 +278,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -340,7 +366,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -430,6 +456,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -507,7 +544,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -561,7 +598,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -578,7 +615,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -595,7 +632,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -612,7 +649,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -629,7 +666,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -646,7 +683,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -665,7 +702,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -688,7 +725,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -703,7 +740,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: grafana-storage: {} diff --git a/deploy/docker-compose/tg-launch-llamafile-cassandra.yaml b/deploy/docker-compose/tg-launch-llamafile-cassandra.yaml new file mode 100644 index 00000000..8886f99c --- /dev/null +++ b/deploy/docker-compose/tg-launch-llamafile-cassandra.yaml @@ -0,0 +1,440 @@ +services: + cassandra: + deploy: + resources: + limits: + cpus: '1.0' + memory: 800M + reservations: + cpus: '0.5' + memory: 800M + environment: + JVM_OPTS: -Xms256M -Xmx256M + image: docker.io/cassandra:4.1.6 + ports: + - 9042:9042 + restart: on-failure:100 + volumes: + - cassandra:/var/lib/cassandra + chunker: + command: + - chunker-token + - -p + - pulsar://pulsar:6650 + - --chunk-size + - '250' + - --chunk-overlap + - '15' + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + embeddings: + command: + - embeddings-hf + - -p + - pulsar://pulsar:6650 + - -m + - all-MiniLM-L6-v2 + deploy: + resources: + limits: + cpus: '1.0' + memory: 400M + reservations: + cpus: '0.5' + memory: 400M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + grafana: + deploy: + resources: + limits: + cpus: '1.0' + memory: 256M + reservations: + cpus: '0.5' + memory: 256M + environment: + GF_ORG_NAME: trustgraph.ai + image: docker.io/grafana/grafana:11.1.4 + ports: + - 3000:3000 + restart: on-failure:100 + volumes: + - grafana-storage:/var/lib/grafana + - ${CONFIGDIR}/grafana/provisioning/:/etc/grafana/provisioning/dashboards/ + - ${CONFIGDIR}/grafana/provisioning/:/etc/grafana/provisioning/datasources/ + - ${CONFIGDIR}/grafana/dashboards/:/var/lib/grafana/dashboards/ + graph-rag: + command: + - graph-rag + - -p + - pulsar://pulsar:6650 + - --prompt-request-queue + - non-persistent://tg/request/prompt-rag + - --prompt-response-queue + - non-persistent://tg/response/prompt-rag-response + - --entity-limit + - '50' + - --triple-limit + - '30' + - --max-subgraph-size + - '3000' + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + init-pulsar: + command: + - sh + - -c + - while true; do pulsar-admin --admin-url http://pulsar:8080 tenants create tg + ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow ; pulsar-admin + --admin-url http://pulsar:8080 namespaces create tg/request ; pulsar-admin --admin-url + http://pulsar:8080 namespaces create tg/response ; pulsar-admin --admin-url + http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response; + sleep 20; done + deploy: + resources: + limits: + cpus: '1' + memory: 400M + reservations: + cpus: '0.1' + memory: 400M + image: docker.io/apachepulsar/pulsar:3.3.1 + restart: on-failure:100 + kg-extract-definitions: + command: + - kg-extract-definitions + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-relationships: + command: + - kg-extract-relationships + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + pdf-decoder: + command: + - pdf-decoder + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + prometheus: + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/prom/prometheus:v2.53.2 + ports: + - 9090:9090 + restart: on-failure:100 + volumes: + - ${CONFIGDIR}/prometheus:/etc/prometheus/ + - prometheus-data:/prometheus + prompt: + command: + - prompt-generic + - -p + - pulsar://pulsar:6650 + - --text-completion-request-queue + - non-persistent://tg/request/text-completion + - --text-completion-response-queue + - non-persistent://tg/response/text-completion-response + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + prompt-rag: + command: + - prompt-generic + - -p + - pulsar://pulsar:6650 + - -i + - non-persistent://tg/request/prompt-rag + - -o + - non-persistent://tg/response/prompt-rag-response + - --text-completion-request-queue + - non-persistent://tg/request/text-completion-rag + - --text-completion-response-queue + - non-persistent://tg/response/text-completion-rag-response + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + pulsar: + command: + - bin/pulsar + - standalone + deploy: + resources: + limits: + cpus: '2.0' + memory: 1500M + reservations: + cpus: '1.0' + memory: 1500M + environment: + PULSAR_MEM: -Xms600M -Xmx600M + image: docker.io/apachepulsar/pulsar:3.3.1 + ports: + - 6650:6650 + - 8080:8080 + restart: on-failure:100 + volumes: + - pulsar-data:/pulsar/data + qdrant: + deploy: + resources: + limits: + cpus: '1.0' + memory: 256M + reservations: + cpus: '0.5' + memory: 256M + image: docker.io/qdrant/qdrant:v1.11.1 + ports: + - 6333:6333 + - 6334:6334 + restart: on-failure:100 + volumes: + - qdrant:/qdrant/storage + query-doc-embeddings: + command: + - de-query-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + query-graph-embeddings: + command: + - ge-query-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + query-triples: + command: + - triples-query-cassandra + - -p + - pulsar://pulsar:6650 + - -g + - cassandra + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + reservations: + cpus: '0.1' + memory: 512M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + store-doc-embeddings: + command: + - de-write-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + store-graph-embeddings: + command: + - ge-write-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + store-triples: + command: + - triples-write-cassandra + - -p + - pulsar://pulsar:6650 + - -g + - cassandra + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + text-completion: + command: + - text-completion-llamafile + - -p + - pulsar://pulsar:6650 + - -m + - LLaMA_CPP + - -r + - ${LLAMAFILE_URL} + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + text-completion-rag: + command: + - text-completion-llamafile + - -p + - pulsar://pulsar:6650 + - -m + - LLaMA_CPP + - -r + - ${LLAMAFILE_URL} + - -i + - non-persistent://tg/request/text-completion-rag + - -o + - non-persistent://tg/response/text-completion-rag-response + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + vectorize: + command: + - embeddings-vectorize + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '1.0' + memory: 512M + reservations: + cpus: '0.5' + memory: 512M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 +volumes: + cassandra: {} + grafana-storage: {} + prometheus-data: {} + pulsar-data: {} + qdrant: {} + diff --git a/deploy/docker-compose/tg-launch-llamafile-neo4j.yaml b/deploy/docker-compose/tg-launch-llamafile-neo4j.yaml new file mode 100644 index 00000000..41381a80 --- /dev/null +++ b/deploy/docker-compose/tg-launch-llamafile-neo4j.yaml @@ -0,0 +1,441 @@ +services: + chunker: + command: + - chunker-token + - -p + - pulsar://pulsar:6650 + - --chunk-size + - '250' + - --chunk-overlap + - '15' + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + embeddings: + command: + - embeddings-hf + - -p + - pulsar://pulsar:6650 + - -m + - all-MiniLM-L6-v2 + deploy: + resources: + limits: + cpus: '1.0' + memory: 400M + reservations: + cpus: '0.5' + memory: 400M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + grafana: + deploy: + resources: + limits: + cpus: '1.0' + memory: 256M + reservations: + cpus: '0.5' + memory: 256M + environment: + GF_ORG_NAME: trustgraph.ai + image: docker.io/grafana/grafana:11.1.4 + ports: + - 3000:3000 + restart: on-failure:100 + volumes: + - grafana-storage:/var/lib/grafana + - ${CONFIGDIR}/grafana/provisioning/:/etc/grafana/provisioning/dashboards/ + - ${CONFIGDIR}/grafana/provisioning/:/etc/grafana/provisioning/datasources/ + - ${CONFIGDIR}/grafana/dashboards/:/var/lib/grafana/dashboards/ + graph-rag: + command: + - graph-rag + - -p + - pulsar://pulsar:6650 + - --prompt-request-queue + - non-persistent://tg/request/prompt-rag + - --prompt-response-queue + - non-persistent://tg/response/prompt-rag-response + - --entity-limit + - '50' + - --triple-limit + - '30' + - --max-subgraph-size + - '3000' + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + init-pulsar: + command: + - sh + - -c + - while true; do pulsar-admin --admin-url http://pulsar:8080 tenants create tg + ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow ; pulsar-admin + --admin-url http://pulsar:8080 namespaces create tg/request ; pulsar-admin --admin-url + http://pulsar:8080 namespaces create tg/response ; pulsar-admin --admin-url + http://pulsar:8080 namespaces set-retention --size -1 --time 3m tg/response; + sleep 20; done + deploy: + resources: + limits: + cpus: '1' + memory: 400M + reservations: + cpus: '0.1' + memory: 400M + image: docker.io/apachepulsar/pulsar:3.3.1 + restart: on-failure:100 + kg-extract-definitions: + command: + - kg-extract-definitions + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-relationships: + command: + - kg-extract-relationships + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + neo4j: + deploy: + resources: + limits: + cpus: '1.0' + memory: 768M + reservations: + cpus: '0.5' + memory: 768M + environment: + NEO4J_AUTH: neo4j/password + image: docker.io/neo4j:5.22.0-community-bullseye + ports: + - 7474:7474 + - 7687:7687 + restart: on-failure:100 + volumes: + - neo4j:/data + pdf-decoder: + command: + - pdf-decoder + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + prometheus: + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/prom/prometheus:v2.53.2 + ports: + - 9090:9090 + restart: on-failure:100 + volumes: + - ${CONFIGDIR}/prometheus:/etc/prometheus/ + - prometheus-data:/prometheus + prompt: + command: + - prompt-generic + - -p + - pulsar://pulsar:6650 + - --text-completion-request-queue + - non-persistent://tg/request/text-completion + - --text-completion-response-queue + - non-persistent://tg/response/text-completion-response + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + prompt-rag: + command: + - prompt-generic + - -p + - pulsar://pulsar:6650 + - -i + - non-persistent://tg/request/prompt-rag + - -o + - non-persistent://tg/response/prompt-rag-response + - --text-completion-request-queue + - non-persistent://tg/request/text-completion-rag + - --text-completion-response-queue + - non-persistent://tg/response/text-completion-rag-response + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + pulsar: + command: + - bin/pulsar + - standalone + deploy: + resources: + limits: + cpus: '2.0' + memory: 1500M + reservations: + cpus: '1.0' + memory: 1500M + environment: + PULSAR_MEM: -Xms600M -Xmx600M + image: docker.io/apachepulsar/pulsar:3.3.1 + ports: + - 6650:6650 + - 8080:8080 + restart: on-failure:100 + volumes: + - pulsar-data:/pulsar/data + qdrant: + deploy: + resources: + limits: + cpus: '1.0' + memory: 256M + reservations: + cpus: '0.5' + memory: 256M + image: docker.io/qdrant/qdrant:v1.11.1 + ports: + - 6333:6333 + - 6334:6334 + restart: on-failure:100 + volumes: + - qdrant:/qdrant/storage + query-doc-embeddings: + command: + - de-query-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + query-graph-embeddings: + command: + - ge-query-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + query-triples: + command: + - triples-query-neo4j + - -p + - pulsar://pulsar:6650 + - -g + - bolt://neo4j:7687 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + store-doc-embeddings: + command: + - de-write-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + store-graph-embeddings: + command: + - ge-write-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + store-triples: + command: + - triples-write-neo4j + - -p + - pulsar://pulsar:6650 + - -g + - bolt://neo4j:7687 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + text-completion: + command: + - text-completion-llamafile + - -p + - pulsar://pulsar:6650 + - -m + - LLaMA_CPP + - -r + - ${LLAMAFILE_URL} + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + text-completion-rag: + command: + - text-completion-llamafile + - -p + - pulsar://pulsar:6650 + - -m + - LLaMA_CPP + - -r + - ${LLAMAFILE_URL} + - -i + - non-persistent://tg/request/text-completion-rag + - -o + - non-persistent://tg/response/text-completion-rag-response + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + vectorize: + command: + - embeddings-vectorize + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '1.0' + memory: 512M + reservations: + cpus: '0.5' + memory: 512M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 +volumes: + grafana-storage: {} + neo4j: {} + prometheus-data: {} + pulsar-data: {} + qdrant: {} + diff --git a/deploy/docker-compose/tg-launch-ollama-cassandra.yaml b/deploy/docker-compose/tg-launch-ollama-cassandra.yaml index b827199d..6868243e 100644 --- a/deploy/docker-compose/tg-launch-ollama-cassandra.yaml +++ b/deploy/docker-compose/tg-launch-ollama-cassandra.yaml @@ -33,7 +33,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -50,7 +50,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -95,7 +95,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -130,7 +130,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -145,7 +145,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pdf-decoder: command: @@ -160,7 +175,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -195,7 +210,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -218,7 +233,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -272,7 +287,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -289,7 +304,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -306,7 +321,7 @@ services: reservations: cpus: '0.1' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -323,7 +338,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -340,7 +355,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -357,7 +372,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -376,7 +391,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -399,7 +414,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -414,7 +429,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: cassandra: {} diff --git a/deploy/docker-compose/tg-launch-ollama-neo4j.yaml b/deploy/docker-compose/tg-launch-ollama-neo4j.yaml index bad61b37..3a2b0b74 100644 --- a/deploy/docker-compose/tg-launch-ollama-neo4j.yaml +++ b/deploy/docker-compose/tg-launch-ollama-neo4j.yaml @@ -16,7 +16,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -33,7 +33,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -78,7 +78,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -113,7 +113,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -128,7 +128,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 neo4j: deploy: @@ -161,7 +176,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -196,7 +211,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -219,7 +234,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -273,7 +288,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -290,7 +305,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -307,7 +322,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -324,7 +339,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -341,7 +356,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -358,7 +373,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -377,7 +392,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -400,7 +415,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -415,7 +430,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: grafana-storage: {} diff --git a/deploy/docker-compose/tg-launch-openai-cassandra.yaml b/deploy/docker-compose/tg-launch-openai-cassandra.yaml index c498c630..7f2e966f 100644 --- a/deploy/docker-compose/tg-launch-openai-cassandra.yaml +++ b/deploy/docker-compose/tg-launch-openai-cassandra.yaml @@ -33,7 +33,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -50,7 +50,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -95,7 +95,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -130,7 +130,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -145,7 +145,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pdf-decoder: command: @@ -160,7 +175,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -262,6 +277,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -339,7 +365,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -429,6 +455,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -506,7 +543,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -560,7 +597,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -577,7 +614,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -594,7 +631,7 @@ services: reservations: cpus: '0.1' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -611,7 +648,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -628,7 +665,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -645,7 +682,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -668,7 +705,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -695,7 +732,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -710,7 +747,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: cassandra: {} diff --git a/deploy/docker-compose/tg-launch-openai-neo4j.yaml b/deploy/docker-compose/tg-launch-openai-neo4j.yaml index c5577b8f..9d2141a7 100644 --- a/deploy/docker-compose/tg-launch-openai-neo4j.yaml +++ b/deploy/docker-compose/tg-launch-openai-neo4j.yaml @@ -16,7 +16,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -33,7 +33,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -78,7 +78,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -113,7 +113,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -128,7 +128,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 neo4j: deploy: @@ -161,7 +176,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -263,6 +278,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -340,7 +366,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -430,6 +456,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -507,7 +544,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -561,7 +598,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -578,7 +615,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -595,7 +632,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -612,7 +649,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -629,7 +666,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -646,7 +683,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -669,7 +706,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion-rag: command: @@ -696,7 +733,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 vectorize: command: @@ -711,7 +748,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: grafana-storage: {} diff --git a/deploy/docker-compose/tg-launch-vertexai-cassandra.yaml b/deploy/docker-compose/tg-launch-vertexai-cassandra.yaml index 1f7272d7..8a7878ce 100644 --- a/deploy/docker-compose/tg-launch-vertexai-cassandra.yaml +++ b/deploy/docker-compose/tg-launch-vertexai-cassandra.yaml @@ -33,7 +33,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -50,7 +50,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -95,7 +95,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -130,7 +130,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -145,7 +145,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pdf-decoder: command: @@ -160,7 +175,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -262,6 +277,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -339,7 +365,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -429,6 +455,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -506,7 +543,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -560,7 +597,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -577,7 +614,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -594,7 +631,7 @@ services: reservations: cpus: '0.1' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -611,7 +648,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -628,7 +665,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -645,7 +682,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -670,7 +707,7 @@ services: reservations: cpus: '0.1' memory: 256M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: - ${CONFIGDIR}/./vertexai:/vertexai @@ -701,7 +738,7 @@ services: reservations: cpus: '0.1' memory: 256M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: - ${CONFIGDIR}/./vertexai:/vertexai @@ -718,7 +755,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: cassandra: {} diff --git a/deploy/docker-compose/tg-launch-vertexai-neo4j.yaml b/deploy/docker-compose/tg-launch-vertexai-neo4j.yaml index f606bcee..c273da2b 100644 --- a/deploy/docker-compose/tg-launch-vertexai-neo4j.yaml +++ b/deploy/docker-compose/tg-launch-vertexai-neo4j.yaml @@ -16,7 +16,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 embeddings: command: @@ -33,7 +33,7 @@ services: reservations: cpus: '0.5' memory: 400M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 grafana: deploy: @@ -78,7 +78,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 init-pulsar: command: @@ -113,7 +113,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 kg-extract-relationships: command: @@ -128,7 +128,22 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + restart: on-failure:100 + kg-extract-topics: + command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + deploy: + resources: + limits: + cpus: '0.5' + memory: 128M + reservations: + cpus: '0.1' + memory: 128M + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 neo4j: deploy: @@ -161,7 +176,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prometheus: deploy: @@ -263,6 +278,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -340,7 +366,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 prompt-rag: command: @@ -430,6 +456,17 @@ services: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks for\ + \ a provided text.\nRead the provided text. You will identify topics and their\ + \ definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting\ + \ in the provided text.\n- Study the provided text carefully.\n\nHere is the\ + \ text:\n{text}\n\nResponse Instructions: \n- Do not respond with special characters.\n\ + - Return only topics that are concepts and unique to the provided text.\n- Respond\ + \ only with well-formed JSON.\n- The JSON response shall be an array of objects\ + \ with keys \"topic\" and \"definition\". \n- The JSON response shall use the\ + \ following structure:\n\n```json\n[{{\"topic\": string, \"definition\": string}}]\n\ + ```\n\n- Do not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the @@ -507,7 +544,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 pulsar: command: @@ -561,7 +598,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -578,7 +615,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -595,7 +632,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -612,7 +649,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -629,7 +666,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -646,7 +683,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 text-completion: command: @@ -671,7 +708,7 @@ services: reservations: cpus: '0.1' memory: 256M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: - ${CONFIGDIR}/./vertexai:/vertexai @@ -702,7 +739,7 @@ services: reservations: cpus: '0.1' memory: 256M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: - ${CONFIGDIR}/./vertexai:/vertexai @@ -719,7 +756,7 @@ services: reservations: cpus: '0.5' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: grafana-storage: {} diff --git a/deploy/docker-compose/tg-storage-cassandra.yaml b/deploy/docker-compose/tg-storage-cassandra.yaml index c4d90179..2ee7023e 100644 --- a/deploy/docker-compose/tg-storage-cassandra.yaml +++ b/deploy/docker-compose/tg-storage-cassandra.yaml @@ -124,7 +124,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -141,7 +141,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -158,7 +158,7 @@ services: reservations: cpus: '0.1' memory: 512M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -175,7 +175,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -192,7 +192,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -209,7 +209,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: cassandra: {} diff --git a/deploy/docker-compose/tg-storage-neo4j.yaml b/deploy/docker-compose/tg-storage-neo4j.yaml index bb9d858e..5ba4fb02 100644 --- a/deploy/docker-compose/tg-storage-neo4j.yaml +++ b/deploy/docker-compose/tg-storage-neo4j.yaml @@ -125,7 +125,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-graph-embeddings: command: @@ -142,7 +142,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 query-triples: command: @@ -159,7 +159,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-doc-embeddings: command: @@ -176,7 +176,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-graph-embeddings: command: @@ -193,7 +193,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 store-triples: command: @@ -210,7 +210,7 @@ services: reservations: cpus: '0.1' memory: 128M - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 restart: on-failure:100 volumes: grafana-storage: {} diff --git a/deploy/minikube/tg-launch-azure-cassandra.yaml b/deploy/minikube/tg-launch-azure-cassandra.yaml index 5f3c16dd..4cd93a5b 100644 --- a/deploy/minikube/tg-launch-azure-cassandra.yaml +++ b/deploy/minikube/tg-launch-azure-cassandra.yaml @@ -110,7 +110,7 @@ items: - '250' - --chunk-overlap - '15' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -160,7 +160,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -669,7 +669,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -717,7 +717,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -765,7 +765,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -791,6 +791,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: apps/v1 kind: Deployment metadata: @@ -813,7 +848,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -852,11 +887,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1043,6 +1079,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1113,7 +1161,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1244,6 +1292,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1314,7 +1374,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1572,7 +1632,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1622,7 +1682,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1672,7 +1732,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1722,7 +1782,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1772,7 +1832,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1822,7 +1882,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1878,7 +1938,7 @@ items: - '4096' - -t - '0' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1938,7 +1998,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1986,7 +2046,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-azure-neo4j.yaml b/deploy/minikube/tg-launch-azure-neo4j.yaml index 5341546e..739751bf 100644 --- a/deploy/minikube/tg-launch-azure-neo4j.yaml +++ b/deploy/minikube/tg-launch-azure-neo4j.yaml @@ -31,7 +31,7 @@ items: - '250' - --chunk-overlap - '15' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -81,7 +81,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -590,7 +590,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -638,7 +638,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -686,7 +686,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -712,6 +712,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: v1 kind: PersistentVolume metadata: @@ -818,7 +853,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -857,11 +892,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1048,6 +1084,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1118,7 +1166,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1249,6 +1297,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1319,7 +1379,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1577,7 +1637,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1627,7 +1687,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1677,7 +1737,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1727,7 +1787,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1777,7 +1837,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1827,7 +1887,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1883,7 +1943,7 @@ items: - '4096' - -t - '0' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1943,7 +2003,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1991,7 +2051,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-bedrock-cassandra.yaml b/deploy/minikube/tg-launch-bedrock-cassandra.yaml index 9db0d164..a9c6e9a6 100644 --- a/deploy/minikube/tg-launch-bedrock-cassandra.yaml +++ b/deploy/minikube/tg-launch-bedrock-cassandra.yaml @@ -110,7 +110,7 @@ items: - '2000' - --chunk-overlap - '100' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -160,7 +160,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -669,7 +669,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -717,7 +717,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -765,7 +765,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -791,6 +791,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: apps/v1 kind: Deployment metadata: @@ -813,7 +848,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -852,11 +887,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1043,6 +1079,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1113,7 +1161,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1244,6 +1292,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1314,7 +1374,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1572,7 +1632,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1622,7 +1682,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1672,7 +1732,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1722,7 +1782,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1772,7 +1832,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1822,7 +1882,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1882,7 +1942,7 @@ items: - '0' - -m - mistral.mixtral-8x7b-instruct-v0:1 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1946,7 +2006,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1994,7 +2054,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-bedrock-neo4j.yaml b/deploy/minikube/tg-launch-bedrock-neo4j.yaml index 6c464845..8a646d32 100644 --- a/deploy/minikube/tg-launch-bedrock-neo4j.yaml +++ b/deploy/minikube/tg-launch-bedrock-neo4j.yaml @@ -31,7 +31,7 @@ items: - '2000' - --chunk-overlap - '100' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -81,7 +81,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -590,7 +590,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -638,7 +638,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -686,7 +686,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -712,6 +712,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: v1 kind: PersistentVolume metadata: @@ -818,7 +853,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -857,11 +892,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1048,6 +1084,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1118,7 +1166,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1249,6 +1297,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1319,7 +1379,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1577,7 +1637,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1627,7 +1687,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1677,7 +1737,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1727,7 +1787,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1777,7 +1837,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1827,7 +1887,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1887,7 +1947,7 @@ items: - '0' - -m - mistral.mixtral-8x7b-instruct-v0:1 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1951,7 +2011,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1999,7 +2059,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-claude-cassandra.yaml b/deploy/minikube/tg-launch-claude-cassandra.yaml index a49a2dc5..58ef9a6f 100644 --- a/deploy/minikube/tg-launch-claude-cassandra.yaml +++ b/deploy/minikube/tg-launch-claude-cassandra.yaml @@ -110,7 +110,7 @@ items: - '250' - --chunk-overlap - '15' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -160,7 +160,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -669,7 +669,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -717,7 +717,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -765,7 +765,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -791,6 +791,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: apps/v1 kind: Deployment metadata: @@ -813,7 +848,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -852,11 +887,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1043,6 +1079,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1113,7 +1161,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1244,6 +1292,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1314,7 +1374,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1572,7 +1632,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1622,7 +1682,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1672,7 +1732,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1722,7 +1782,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1772,7 +1832,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1822,7 +1882,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1876,7 +1936,7 @@ items: - '4096' - -t - '0' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1934,7 +1994,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1982,7 +2042,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-claude-neo4j.yaml b/deploy/minikube/tg-launch-claude-neo4j.yaml index c3608bab..e05cc9df 100644 --- a/deploy/minikube/tg-launch-claude-neo4j.yaml +++ b/deploy/minikube/tg-launch-claude-neo4j.yaml @@ -31,7 +31,7 @@ items: - '250' - --chunk-overlap - '15' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -81,7 +81,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -590,7 +590,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -638,7 +638,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -686,7 +686,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -712,6 +712,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: v1 kind: PersistentVolume metadata: @@ -818,7 +853,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -857,11 +892,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1048,6 +1084,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1118,7 +1166,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1249,6 +1297,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1319,7 +1379,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1577,7 +1637,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1627,7 +1687,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1677,7 +1737,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1727,7 +1787,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1777,7 +1837,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1827,7 +1887,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1881,7 +1941,7 @@ items: - '4096' - -t - '0' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1939,7 +1999,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1987,7 +2047,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-cohere-cassandra.yaml b/deploy/minikube/tg-launch-cohere-cassandra.yaml index f803186c..bf07ba3e 100644 --- a/deploy/minikube/tg-launch-cohere-cassandra.yaml +++ b/deploy/minikube/tg-launch-cohere-cassandra.yaml @@ -110,7 +110,7 @@ items: - '150' - --chunk-overlap - '10' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -160,7 +160,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -669,7 +669,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -717,7 +717,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -765,7 +765,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -791,6 +791,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: apps/v1 kind: Deployment metadata: @@ -813,7 +848,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -852,11 +887,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1043,6 +1079,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1113,7 +1161,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1244,6 +1292,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1314,7 +1374,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1572,7 +1632,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1622,7 +1682,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1672,7 +1732,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1722,7 +1782,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1772,7 +1832,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1822,7 +1882,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1874,7 +1934,7 @@ items: - ${COHERE_KEY} - -t - 0 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1930,7 +1990,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1978,7 +2038,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-cohere-neo4j.yaml b/deploy/minikube/tg-launch-cohere-neo4j.yaml index 5f1aa0be..cc682a7a 100644 --- a/deploy/minikube/tg-launch-cohere-neo4j.yaml +++ b/deploy/minikube/tg-launch-cohere-neo4j.yaml @@ -31,7 +31,7 @@ items: - '150' - --chunk-overlap - '10' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -81,7 +81,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -590,7 +590,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -638,7 +638,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -686,7 +686,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -712,6 +712,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: v1 kind: PersistentVolume metadata: @@ -818,7 +853,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -857,11 +892,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1048,6 +1084,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1118,7 +1166,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1249,6 +1297,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1319,7 +1379,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1577,7 +1637,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1627,7 +1687,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1677,7 +1737,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1727,7 +1787,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1777,7 +1837,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1827,7 +1887,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1879,7 +1939,7 @@ items: - ${COHERE_KEY} - -t - 0 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1935,7 +1995,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1983,7 +2043,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-llamafile-cassandra.yaml b/deploy/minikube/tg-launch-llamafile-cassandra.yaml new file mode 100644 index 00000000..8e9e0d53 --- /dev/null +++ b/deploy/minikube/tg-launch-llamafile-cassandra.yaml @@ -0,0 +1,1728 @@ +apiVersion: v1 +items: +- apiVersion: v1 + kind: Namespace + metadata: + name: trustgraph + spec: {} +- apiVersion: v1 + kind: PersistentVolume + metadata: + name: cassandra + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 20G + hostPath: + path: /data/pv-cassandra + persistentVolumeReclaimPolicy: Delete +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: cassandra + namespace: trustgraph + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20G +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: cassandra + name: cassandra + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: cassandra + template: + metadata: + labels: + app: cassandra + spec: + containers: + - env: + - name: JVM_OPTS + value: -Xms256M -Xmx256M + image: docker.io/cassandra:4.1.6 + name: cassandra + ports: + - containerPort: 9042 + hostPort: 9042 + resources: + limits: + cpu: '1.0' + memory: 800M + requests: + cpu: '0.5' + memory: 800M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /var/lib/cassandra + name: cassandra + volumes: + - name: cassandra + persistentVolumeClaim: + claimName: cassandra +- apiVersion: v1 + kind: Service + metadata: + name: cassandra + namespace: trustgraph + spec: + ports: + - name: api + port: 9042 + targetPort: 9042 + selector: + app: cassandra + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: chunker + name: chunker + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: chunker + template: + metadata: + labels: + app: chunker + spec: + containers: + - command: + - chunker-token + - -p + - pulsar://pulsar:6650 + - --chunk-size + - '250' + - --chunk-overlap + - '15' + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: chunker + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: chunker + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: chunker + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: embeddings + name: embeddings + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: embeddings + template: + metadata: + labels: + app: embeddings + spec: + containers: + - command: + - embeddings-hf + - -p + - pulsar://pulsar:6650 + - -m + - all-MiniLM-L6-v2 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: embeddings + resources: + limits: + cpu: '1.0' + memory: 400M + requests: + cpu: '0.5' + memory: 400M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: embeddings + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: embeddings + type: LoadBalancer +- apiVersion: v1 + kind: PersistentVolume + metadata: + name: grafana-storage + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 20G + hostPath: + path: /data/pv-grafana-storage + persistentVolumeReclaimPolicy: Delete +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: grafana-storage + namespace: trustgraph + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20G +- apiVersion: v1 + data: + dashboard.yml: "\napiVersion: 1\n\nproviders:\n\n - name: 'trustgraph.ai'\n \ + \ orgId: 1\n folder: 'TrustGraph'\n folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228'\n\ + \ type: file\n disableDeletion: false\n updateIntervalSeconds: 30\n\ + \ allowUiUpdates: true\n options:\n path: /var/lib/grafana/dashboards\n\ + \ foldersFromFilesStructure: false\n\n" + kind: ConfigMap + metadata: + name: prov-dash + namespace: trustgraph +- apiVersion: v1 + data: + datasource.yml: "apiVersion: 1\n\nprune: true\n\ndatasources:\n - name: Prometheus\n\ + \ type: prometheus\n access: proxy\n orgId: 1\n # Sets\ + \ a custom UID to reference this\n # data source in other parts of the configuration.\n\ + \ # If not specified, Grafana generates one.\n uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129'\n\ + \n url: http://prometheus:9090\n\n basicAuth: false\n withCredentials:\ + \ false\n isDefault: true\n editable: true\n\n" + kind: ConfigMap + metadata: + name: prov-data + namespace: trustgraph +- apiVersion: v1 + data: + dashboard.json: "{\n \"annotations\": {\n \"list\": [\n {\n \"\ + builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n\ + \ \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n\ + \ \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n\ + \ \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\ + \n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n\ + \ \"graphTooltip\": 0,\n \"id\": 2,\n \"links\": [],\n \"liveNow\": false,\n\ + \ \"panels\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\"\ + ,\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n },\n \ + \ \"fieldConfig\": {\n \"defaults\": {\n \"custom\": {\n \ + \ \"hideFrom\": {\n \"legend\": false,\n \ + \ \"tooltip\": false,\n \"viz\": false\n },\n \ + \ \"scaleDistribution\": {\n \"type\": \"linear\"\n \ + \ }\n }\n },\n \"overrides\": []\n },\n \ + \ \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n\ + \ \"y\": 0\n },\n \"id\": 7,\n \"options\": {\n \ + \ \"calculate\": false,\n \"cellGap\": 1,\n \"color\": {\n \ + \ \"exponent\": 0.5,\n \"fill\": \"dark-orange\",\n \ + \ \"mode\": \"scheme\",\n \"reverse\": false,\n \"scale\"\ + : \"exponential\",\n \"scheme\": \"Oranges\",\n \"steps\"\ + : 64\n },\n \"exemplars\": {\n \"color\": \"rgba(255,0,255,0.7)\"\ + \n },\n \"filterValues\": {\n \"le\": 1e-9\n },\n\ + \ \"legend\": {\n \"show\": true\n },\n \"rowsFrame\"\ + : {\n \"layout\": \"auto\"\n },\n \"tooltip\": {\n \ + \ \"mode\": \"single\",\n \"showColorScale\": false,\n \ + \ \"yHistogram\": false\n },\n \"yAxis\": {\n \"\ + axisPlacement\": \"left\",\n \"reverse\": false\n }\n },\n\ + \ \"pluginVersion\": \"11.1.4\",\n \"targets\": [\n {\n \ + \ \"datasource\": {\n \"type\": \"prometheus\",\n \ + \ \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n },\n \ + \ \"editorMode\": \"code\",\n \"exemplar\": false,\n \"\ + expr\": \"max by(le) (text_completion_duration_bucket)\",\n \"format\"\ + : \"heatmap\",\n \"instant\": false,\n \"legendFormat\": \"\ + 99%\",\n \"range\": true,\n \"refId\": \"A\"\n }\n\ + \ ],\n \"title\": \"LLM latency\",\n \"type\": \"heatmap\"\n\ + \ },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n\ + \ \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n },\n \ + \ \"fieldConfig\": {\n \"defaults\": {\n \"custom\": {\n \ + \ \"hideFrom\": {\n \"legend\": false,\n \"\ + tooltip\": false,\n \"viz\": false\n },\n \ + \ \"scaleDistribution\": {\n \"type\": \"linear\"\n \ + \ }\n }\n },\n \"overrides\": []\n },\n \"\ + gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \ + \ \"y\": 0\n },\n \"id\": 2,\n \"options\": {\n \"\ + calculate\": false,\n \"cellGap\": 5,\n \"cellValues\": {\n \ + \ \"unit\": \"\"\n },\n \"color\": {\n \"exponent\"\ + : 0.5,\n \"fill\": \"dark-orange\",\n \"mode\": \"scheme\"\ + ,\n \"reverse\": false,\n \"scale\": \"exponential\",\n \ + \ \"scheme\": \"Oranges\",\n \"steps\": 64\n },\n \ + \ \"exemplars\": {\n \"color\": \"rgba(255,0,255,0.7)\"\n \ + \ },\n \"filterValues\": {\n \"le\": 1e-9\n },\n \ + \ \"legend\": {\n \"show\": true\n },\n \"rowsFrame\"\ + : {\n \"layout\": \"auto\"\n },\n \"tooltip\": {\n \ + \ \"mode\": \"single\",\n \"showColorScale\": false,\n \ + \ \"yHistogram\": false\n },\n \"yAxis\": {\n \"\ + axisLabel\": \"processing status\",\n \"axisPlacement\": \"left\",\n\ + \ \"reverse\": false\n }\n },\n \"pluginVersion\"\ + : \"11.1.4\",\n \"targets\": [\n {\n \"datasource\": {\n\ + \ \"type\": \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"disableTextWrap\": false,\n \"editorMode\"\ + : \"builder\",\n \"exemplar\": false,\n \"expr\": \"sum by(status)\ + \ (rate(processing_count_total{status!=\\\"success\\\"}[$__rate_interval]))\"\ + ,\n \"format\": \"heatmap\",\n \"fullMetaSearch\": false,\n\ + \ \"includeNullMetadata\": true,\n \"instant\": false,\n \ + \ \"interval\": \"\",\n \"legendFormat\": \"{{status}}\",\n\ + \ \"range\": true,\n \"refId\": \"A\",\n \"useBackend\"\ + : false\n }\n ],\n \"title\": \"Error rate\",\n \"type\"\ + : \"heatmap\"\n },\n {\n \"datasource\": {\n \"type\": \"\ + prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n \ + \ },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\"\ + : {\n \"mode\": \"palette-classic\"\n },\n \"custom\"\ + : {\n \"axisBorderShow\": false,\n \"axisCenteredZero\"\ + : false,\n \"axisColorMode\": \"text\",\n \"axisLabel\"\ + : \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\"\ + : 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n\ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \ + \ \"viz\": false\n },\n \"insertNulls\": false,\n\ + \ \"lineInterpolation\": \"linear\",\n \"lineWidth\":\ + \ 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n\ + \ \"type\": \"linear\"\n },\n \"showPoints\"\ + : \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n\ + \ \"group\": \"A\",\n \"mode\": \"none\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\ + \n }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 9,\n \"w\": 12,\n \"x\": 0,\n \"y\": 8\n\ + \ },\n \"id\": 1,\n \"options\": {\n \"legend\": {\n \ + \ \"calcs\": [],\n \"displayMode\": \"list\",\n \"\ + placement\": \"bottom\",\n \"showLegend\": true\n },\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"\ + none\"\n }\n },\n \"targets\": [\n {\n \"datasource\"\ + : {\n \"type\": \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"editorMode\": \"builder\",\n \"expr\":\ + \ \"rate(request_latency_count[1m])\",\n \"instant\": false,\n \ + \ \"legendFormat\": \"{{instance}}\",\n \"range\": true,\n \ + \ \"refId\": \"A\"\n }\n ],\n \"title\": \"Request rate\"\ + ,\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n\ + \ \"type\": \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"mode\": \"palette-classic\"\n },\n \ + \ \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\"\ + : false,\n \"axisColorMode\": \"text\",\n \"axisLabel\"\ + : \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\"\ + : 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n\ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \ + \ \"viz\": false\n },\n \"insertNulls\": false,\n\ + \ \"lineInterpolation\": \"linear\",\n \"lineWidth\":\ + \ 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n\ + \ \"type\": \"linear\"\n },\n \"showPoints\"\ + : \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n\ + \ \"group\": \"A\",\n \"mode\": \"none\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\ + \n }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 9,\n \"w\": 12,\n \"x\": 12,\n \"y\": 8\n\ + \ },\n \"id\": 5,\n \"options\": {\n \"legend\": {\n \ + \ \"calcs\": [],\n \"displayMode\": \"list\",\n \"\ + placement\": \"bottom\",\n \"showLegend\": true\n },\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"\ + none\"\n }\n },\n \"pluginVersion\": \"10.0.0\",\n \"\ + targets\": [\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"editorMode\": \"builder\",\n \"expr\":\ + \ \"pulsar_msg_backlog\",\n \"instant\": false,\n \"legendFormat\"\ + : \"{{topic}}\",\n \"range\": true,\n \"refId\": \"A\"\n \ + \ }\n ],\n \"title\": \"Pub/sub backlog\",\n \"type\":\ + \ \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"fixedColor\": \"semi-dark-green\",\n \"\ + mode\": \"palette-classic-by-name\"\n },\n \"custom\": {\n\ + \ \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n\ + \ \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n\ + \ \"axisPlacement\": \"auto\",\n \"fillOpacity\": 80,\n\ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \ + \ \"viz\": false\n },\n \"lineWidth\": 1,\n \ + \ \"scaleDistribution\": {\n \"type\": \"linear\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\ + \n }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 7,\n \"w\": 12,\n \"x\": 0,\n \"y\": 17\n\ + \ },\n \"id\": 10,\n \"options\": {\n \"barRadius\": 0,\n\ + \ \"barWidth\": 0.97,\n \"fullHighlight\": false,\n \"\ + groupWidth\": 0.7,\n \"legend\": {\n \"calcs\": [],\n \ + \ \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \ + \ \"showLegend\": true\n },\n \"orientation\": \"auto\",\n\ + \ \"showValue\": \"auto\",\n \"stacking\": \"none\",\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\ + \n },\n \"xTickLabelRotation\": 0,\n \"xTickLabelSpacing\"\ + : 0\n },\n \"pluginVersion\": \"11.1.4\",\n \"targets\": [\n\ + \ {\n \"datasource\": {\n \"type\": \"prometheus\"\ + ,\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n \ + \ },\n \"disableTextWrap\": false,\n \"editorMode\": \"builder\"\ + ,\n \"exemplar\": false,\n \"expr\": \"max by(le) (chunk_size_bucket)\"\ + ,\n \"format\": \"heatmap\",\n \"fullMetaSearch\": false,\n\ + \ \"includeNullMetadata\": false,\n \"instant\": true,\n \ + \ \"legendFormat\": \"{{le}}\",\n \"range\": false,\n \ + \ \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n\ + \ \"title\": \"Chunk size\",\n \"type\": \"barchart\"\n },\n \ + \ {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"\ + uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n },\n \"fieldConfig\"\ + : {\n \"defaults\": {\n \"color\": {\n \"mode\":\ + \ \"palette-classic\"\n },\n \"custom\": {\n \"\ + axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \ + \ \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \ + \ \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \ + \ \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \"\ + viz\": false\n },\n \"insertNulls\": false,\n \ + \ \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \ + \ \"pointSize\": 5,\n \"scaleDistribution\": {\n \ + \ \"type\": \"linear\"\n },\n \"showPoints\": \"\ + auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \ + \ \"group\": \"A\",\n \"mode\": \"none\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n\ + \ }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 7,\n \"w\": 12,\n \"x\": 12,\n \"y\": 17\n\ + \ },\n \"id\": 11,\n \"options\": {\n \"legend\": {\n\ + \ \"calcs\": [],\n \"displayMode\": \"list\",\n \"\ + placement\": \"bottom\",\n \"showLegend\": true\n },\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"\ + none\"\n }\n },\n \"pluginVersion\": \"11.1.4\",\n \"\ + targets\": [\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"disableTextWrap\": false,\n \"editorMode\"\ + : \"builder\",\n \"exemplar\": false,\n \"expr\": \"increase(processing_count_total{status!=\\\ + \"success\\\"}[$__rate_interval])\",\n \"format\": \"time_series\"\ + ,\n \"fullMetaSearch\": false,\n \"includeNullMetadata\":\ + \ true,\n \"instant\": false,\n \"legendFormat\": \"{{instance}}\"\ + ,\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\"\ + : false\n }\n ],\n \"title\": \"Errors\",\n \"type\":\ + \ \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"fixedColor\": \"light-blue\",\n \"mode\"\ + : \"palette-classic\"\n },\n \"custom\": {\n \"\ + axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \ + \ \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \ + \ \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \ + \ \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \"\ + viz\": false\n },\n \"insertNulls\": false,\n \ + \ \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \ + \ \"pointSize\": 5,\n \"scaleDistribution\": {\n \ + \ \"type\": \"linear\"\n },\n \"showPoints\": \"\ + auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \ + \ \"group\": \"A\",\n \"mode\": \"none\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n\ + \ }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 24\n\ + \ },\n \"id\": 12,\n \"options\": {\n \"legend\": {\n\ + \ \"calcs\": [],\n \"displayMode\": \"list\",\n \"\ + placement\": \"bottom\",\n \"showLegend\": true\n },\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"\ + none\"\n }\n },\n \"pluginVersion\": \"11.1.4\",\n \"\ + targets\": [\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"disableTextWrap\": false,\n \"editorMode\"\ + : \"builder\",\n \"expr\": \"rate(process_cpu_seconds_total[$__rate_interval])\"\ + ,\n \"fullMetaSearch\": false,\n \"includeNullMetadata\":\ + \ true,\n \"instant\": false,\n \"legendFormat\": \"{{instance}}\"\ + ,\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\"\ + : false\n }\n ],\n \"title\": \"CPU\",\n \"type\": \"\ + timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\"\ + ,\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n },\n \ + \ \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \ + \ \"mode\": \"palette-classic\"\n },\n \"custom\"\ + : {\n \"axisBorderShow\": false,\n \"axisCenteredZero\"\ + : false,\n \"axisColorMode\": \"text\",\n \"axisLabel\"\ + : \"GB\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\"\ + : 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n\ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \ + \ \"viz\": false\n },\n \"insertNulls\": false,\n\ + \ \"lineInterpolation\": \"linear\",\n \"lineWidth\":\ + \ 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n\ + \ \"type\": \"linear\"\n },\n \"showPoints\"\ + : \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n\ + \ \"group\": \"A\",\n \"mode\": \"none\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\ + \n }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 24\n\ + \ },\n \"id\": 13,\n \"options\": {\n \"legend\": {\n\ + \ \"calcs\": [],\n \"displayMode\": \"list\",\n \"\ + placement\": \"bottom\",\n \"showLegend\": true\n },\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"\ + none\"\n }\n },\n \"targets\": [\n {\n \"datasource\"\ + : {\n \"type\": \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"disableTextWrap\": false,\n \"editorMode\"\ + : \"builder\",\n \"expr\": \"process_resident_memory_bytes / 1073741824\"\ + ,\n \"fullMetaSearch\": false,\n \"includeNullMetadata\":\ + \ true,\n \"instant\": false,\n \"legendFormat\": \"{{instance}}\"\ + ,\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\"\ + : false\n }\n ],\n \"title\": \"Memory\",\n \"type\":\ + \ \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"mode\": \"thresholds\"\n },\n \"\ + custom\": {\n \"align\": \"auto\",\n \"cellOptions\":\ + \ {\n \"type\": \"auto\"\n },\n \"filterable\"\ + : false,\n \"inspect\": false\n },\n \"mappings\"\ + : [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"green\"\ + ,\n \"value\": null\n },\n {\n \ + \ \"color\": \"red\",\n \"value\": 80\n \ + \ }\n ]\n }\n },\n \"overrides\": []\n\ + \ },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \ + \ \"x\": 0,\n \"y\": 32\n },\n \"id\": 14,\n \"options\"\ + : {\n \"cellHeight\": \"sm\",\n \"footer\": {\n \"countRows\"\ + : false,\n \"fields\": \"\",\n \"reducer\": [\n \ + \ \"sum\"\n ],\n \"show\": false\n },\n \"\ + showHeader\": true\n },\n \"pluginVersion\": \"11.1.4\",\n \"\ + targets\": [\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"disableTextWrap\": false,\n \"editorMode\"\ + : \"builder\",\n \"exemplar\": false,\n \"expr\": \"last_over_time(params_info[$__interval])\"\ + ,\n \"format\": \"table\",\n \"fullMetaSearch\": false,\n\ + \ \"includeNullMetadata\": true,\n \"instant\": true,\n \ + \ \"legendFormat\": \"__auto\",\n \"range\": false,\n \ + \ \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n\ + \ \"title\": \"Model parameters\",\n \"transformations\": [\n \ + \ {\n \"id\": \"filterFieldsByName\",\n \"options\": {\n\ + \ \"include\": {\n \"names\": [\n \"\ + instance\",\n \"model\"\n ]\n }\n \ + \ }\n },\n {\n \"id\": \"filterByValue\",\n \ + \ \"options\": {\n \"filters\": [\n {\n \ + \ \"config\": {\n \"id\": \"equal\",\n \ + \ \"options\": {\n \"value\": \"\"\n \ + \ }\n },\n \"fieldName\": \"model\"\n \ + \ }\n ],\n \"match\": \"all\",\n \"\ + type\": \"exclude\"\n }\n }\n ],\n \"type\": \"table\"\ + \n }\n ],\n \"refresh\": \"5s\",\n \"schemaVersion\": 39,\n \"tags\"\ + : [],\n \"templating\": {\n \"list\": []\n },\n \"time\": {\n \"from\"\ + : \"now-5m\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\"\ + : \"\",\n \"title\": \"Overview\",\n \"uid\": \"b5c8abf8-fe79-496b-b028-10bde917d1f0\"\ + ,\n \"version\": 3,\n \"weekStart\": \"\"\n}\n" + kind: ConfigMap + metadata: + name: dashboards + namespace: trustgraph +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: grafana + name: grafana + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + containers: + - env: + - name: GF_ORG_NAME + value: trustgraph.ai + image: docker.io/grafana/grafana:11.1.4 + name: grafana + ports: + - containerPort: 3000 + hostPort: 3000 + resources: + limits: + cpu: '1.0' + memory: 256M + requests: + cpu: '0.5' + memory: 256M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /var/lib/grafana + name: grafana-storage + - mountPath: /etc/grafana/provisioning/dashboards/ + name: prov-dash + - mountPath: /etc/grafana/provisioning/datasources/ + name: prov-data + - mountPath: /var/lib/grafana/dashboards/ + name: dashboards + volumes: + - name: grafana-storage + persistentVolumeClaim: + claimName: grafana-storage + - configMap: + name: prov-dash + name: prov-dash + - configMap: + name: prov-data + name: prov-data + - configMap: + name: dashboards + name: dashboards +- apiVersion: v1 + kind: Service + metadata: + name: grafana + namespace: trustgraph + spec: + ports: + - name: http + port: 3000 + targetPort: 3000 + selector: + app: grafana + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: graph-rag + name: graph-rag + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: graph-rag + template: + metadata: + labels: + app: graph-rag + spec: + containers: + - command: + - graph-rag + - -p + - pulsar://pulsar:6650 + - --prompt-request-queue + - non-persistent://tg/request/prompt-rag + - --prompt-response-queue + - non-persistent://tg/response/prompt-rag-response + - --entity-limit + - '50' + - --triple-limit + - '30' + - --max-subgraph-size + - '3000' + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: graph-rag + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: graph-rag + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: graph-rag + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-definitions + name: kg-extract-definitions + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-definitions + template: + metadata: + labels: + app: kg-extract-definitions + spec: + containers: + - command: + - kg-extract-definitions + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-definitions + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: kg-extract-definitions + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: kg-extract-definitions + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-relationships + name: kg-extract-relationships + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-relationships + template: + metadata: + labels: + app: kg-extract-relationships + spec: + containers: + - command: + - kg-extract-relationships + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-relationships + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: kg-extract-relationships + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: kg-extract-relationships + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: pdf-decoder + name: pdf-decoder + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: pdf-decoder + template: + metadata: + labels: + app: pdf-decoder + spec: + containers: + - command: + - pdf-decoder + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: pdf-decoder + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: pdf-decoder + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: pdf-decoder + type: LoadBalancer +- apiVersion: v1 + data: + prometheus.yml: "global:\n\n scrape_interval: 15s # By default, scrape targets\ + \ every 15 seconds.\n\n # Attach these labels to any time series or alerts\ + \ when communicating with\n # external systems (federation, remote storage,\ + \ Alertmanager).\n external_labels:\n monitor: 'trustgraph'\n\n# A scrape\ + \ configuration containing exactly one endpoint to scrape:\n# Here it's Prometheus\ + \ itself.\nscrape_configs:\n\n # The job name is added as a label `job=`\ + \ to any timeseries\n # scraped from this config.\n\n - job_name: 'trustgraph'\n\ + \n # Override the global default and scrape targets from this job every\n\ + \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ + \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ + \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" + kind: ConfigMap + metadata: + name: prometheus-cfg + namespace: trustgraph +- apiVersion: v1 + kind: PersistentVolume + metadata: + name: prometheus-data + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 20G + hostPath: + path: /data/pv-prometheus-data + persistentVolumeReclaimPolicy: Delete +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: prometheus-data + namespace: trustgraph + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20G +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: prometheus + name: prometheus + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + containers: + - image: docker.io/prom/prometheus:v2.53.2 + name: prometheus + ports: + - containerPort: 9090 + hostPort: 9090 + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /etc/prometheus/ + name: prometheus-cfg + - mountPath: /prometheus + name: prometheus-data + volumes: + - configMap: + name: prometheus-cfg + name: prometheus-cfg + - name: prometheus-data + persistentVolumeClaim: + claimName: prometheus-data +- apiVersion: v1 + kind: Service + metadata: + name: prometheus + namespace: trustgraph + spec: + ports: + - name: http + port: 9090 + targetPort: 9090 + selector: + app: prometheus + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: prompt + name: prompt + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: prompt + template: + metadata: + labels: + app: prompt + spec: + containers: + - command: + - prompt-generic + - -p + - pulsar://pulsar:6650 + - --text-completion-request-queue + - non-persistent://tg/request/text-completion + - --text-completion-response-queue + - non-persistent://tg/response/text-completion-response + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: prompt + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: prompt + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: prompt + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: prompt-rag + name: prompt-rag + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: prompt-rag + template: + metadata: + labels: + app: prompt-rag + spec: + containers: + - command: + - prompt-generic + - -p + - pulsar://pulsar:6650 + - -i + - non-persistent://tg/request/prompt-rag + - -o + - non-persistent://tg/response/prompt-rag-response + - --text-completion-request-queue + - non-persistent://tg/request/text-completion-rag + - --text-completion-response-queue + - non-persistent://tg/response/text-completion-rag-response + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: prompt-rag + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: prompt-rag + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: prompt-rag + type: LoadBalancer +- apiVersion: v1 + kind: PersistentVolume + metadata: + name: pulsar-data + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 20G + hostPath: + path: /data/pv-pulsar-data + persistentVolumeReclaimPolicy: Delete +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: pulsar-data + namespace: trustgraph + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20G +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: pulsar + name: pulsar + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: pulsar + template: + metadata: + labels: + app: pulsar + spec: + containers: + - command: + - bin/pulsar + - standalone + env: + - name: PULSAR_MEM + value: -Xms600M -Xmx600M + image: docker.io/apachepulsar/pulsar:3.3.1 + name: pulsar + ports: + - containerPort: 6650 + hostPort: 6650 + - containerPort: 8080 + hostPort: 8080 + resources: + limits: + cpu: '2.0' + memory: 1500M + requests: + cpu: '1.0' + memory: 1500M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /pulsar/data + name: pulsar-data + volumes: + - name: pulsar-data + persistentVolumeClaim: + claimName: pulsar-data +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: init-pulsar + name: init-pulsar + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: init-pulsar + template: + metadata: + labels: + app: init-pulsar + spec: + containers: + - command: + - sh + - -c + - while true; do pulsar-admin --admin-url http://pulsar:8080 tenants create + tg ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow + ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request + ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response + ; pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention + --size -1 --time 3m tg/response; sleep 20; done + image: docker.io/apachepulsar/pulsar:3.3.1 + name: init-pulsar + resources: + limits: + cpu: '1' + memory: 400M + requests: + cpu: '0.1' + memory: 400M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: pulsar + namespace: trustgraph + spec: + ports: + - name: bookie + port: 6650 + targetPort: 6650 + - name: http + port: 8080 + targetPort: 8080 + selector: + app: pulsar + type: LoadBalancer +- apiVersion: v1 + kind: PersistentVolume + metadata: + name: qdrant + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 20G + hostPath: + path: /data/pv-qdrant + persistentVolumeReclaimPolicy: Delete +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: qdrant + namespace: trustgraph + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20G +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: qdrant + name: qdrant + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: qdrant + template: + metadata: + labels: + app: qdrant + spec: + containers: + - image: docker.io/qdrant/qdrant:v1.11.1 + name: qdrant + ports: + - containerPort: 6333 + hostPort: 6333 + - containerPort: 6334 + hostPort: 6334 + resources: + limits: + cpu: '1.0' + memory: 256M + requests: + cpu: '0.5' + memory: 256M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /qdrant/storage + name: qdrant + volumes: + - name: qdrant + persistentVolumeClaim: + claimName: qdrant +- apiVersion: v1 + kind: Service + metadata: + name: qdrant + namespace: trustgraph + spec: + ports: + - name: api + port: 6333 + targetPort: 6333 + - name: api2 + port: 6334 + targetPort: 6334 + selector: + app: qdrant + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: query-doc-embeddings + name: query-doc-embeddings + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: query-doc-embeddings + template: + metadata: + labels: + app: query-doc-embeddings + spec: + containers: + - command: + - de-query-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: query-doc-embeddings + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: query-doc-embeddings + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: query-doc-embeddings + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: query-graph-embeddings + name: query-graph-embeddings + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: query-graph-embeddings + template: + metadata: + labels: + app: query-graph-embeddings + spec: + containers: + - command: + - ge-query-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: query-graph-embeddings + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: query-graph-embeddings + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: query-graph-embeddings + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: query-triples + name: query-triples + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: query-triples + template: + metadata: + labels: + app: query-triples + spec: + containers: + - command: + - triples-query-cassandra + - -p + - pulsar://pulsar:6650 + - -g + - cassandra + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: query-triples + resources: + limits: + cpu: '0.5' + memory: 512M + requests: + cpu: '0.1' + memory: 512M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: query-triples + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: query-triples + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: store-doc-embeddings + name: store-doc-embeddings + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: store-doc-embeddings + template: + metadata: + labels: + app: store-doc-embeddings + spec: + containers: + - command: + - de-write-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: store-doc-embeddings + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: store-doc-embeddings + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: store-doc-embeddings + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: store-graph-embeddings + name: store-graph-embeddings + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: store-graph-embeddings + template: + metadata: + labels: + app: store-graph-embeddings + spec: + containers: + - command: + - ge-write-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: store-graph-embeddings + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: store-graph-embeddings + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: store-graph-embeddings + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: store-triples + name: store-triples + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: store-triples + template: + metadata: + labels: + app: store-triples + spec: + containers: + - command: + - triples-write-cassandra + - -p + - pulsar://pulsar:6650 + - -g + - cassandra + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: store-triples + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: stop-triples + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: stop-triples + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: text-completion + name: text-completion + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: text-completion + template: + metadata: + labels: + app: text-completion + spec: + containers: + - command: + - text-completion-llamafile + - -p + - pulsar://pulsar:6650 + - -m + - LLaMA_CPP + - -r + - ${LLAMAFILE_URL} + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: text-completion + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: text-completion-rag + name: text-completion-rag + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: text-completion-rag + template: + metadata: + labels: + app: text-completion-rag + spec: + containers: + - command: + - text-completion-llamafile + - -p + - pulsar://pulsar:6650 + - -m + - LLaMA_CPP + - -r + - ${LLAMAFILE_URL} + - -i + - non-persistent://tg/request/text-completion-rag + - -o + - non-persistent://tg/response/text-completion-rag-response + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: text-completion-rag + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: vectorize + name: vectorize + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: vectorize + template: + metadata: + labels: + app: vectorize + spec: + containers: + - command: + - embeddings-vectorize + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: vectorize + resources: + limits: + cpu: '1.0' + memory: 512M + requests: + cpu: '0.5' + memory: 512M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: vectorize + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: vectorize + type: LoadBalancer +kind: List + diff --git a/deploy/minikube/tg-launch-llamafile-neo4j.yaml b/deploy/minikube/tg-launch-llamafile-neo4j.yaml new file mode 100644 index 00000000..017d0e57 --- /dev/null +++ b/deploy/minikube/tg-launch-llamafile-neo4j.yaml @@ -0,0 +1,1733 @@ +apiVersion: v1 +items: +- apiVersion: v1 + kind: Namespace + metadata: + name: trustgraph + spec: {} +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: chunker + name: chunker + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: chunker + template: + metadata: + labels: + app: chunker + spec: + containers: + - command: + - chunker-token + - -p + - pulsar://pulsar:6650 + - --chunk-size + - '250' + - --chunk-overlap + - '15' + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: chunker + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: chunker + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: chunker + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: embeddings + name: embeddings + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: embeddings + template: + metadata: + labels: + app: embeddings + spec: + containers: + - command: + - embeddings-hf + - -p + - pulsar://pulsar:6650 + - -m + - all-MiniLM-L6-v2 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: embeddings + resources: + limits: + cpu: '1.0' + memory: 400M + requests: + cpu: '0.5' + memory: 400M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: embeddings + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: embeddings + type: LoadBalancer +- apiVersion: v1 + kind: PersistentVolume + metadata: + name: grafana-storage + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 20G + hostPath: + path: /data/pv-grafana-storage + persistentVolumeReclaimPolicy: Delete +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: grafana-storage + namespace: trustgraph + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20G +- apiVersion: v1 + data: + dashboard.yml: "\napiVersion: 1\n\nproviders:\n\n - name: 'trustgraph.ai'\n \ + \ orgId: 1\n folder: 'TrustGraph'\n folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228'\n\ + \ type: file\n disableDeletion: false\n updateIntervalSeconds: 30\n\ + \ allowUiUpdates: true\n options:\n path: /var/lib/grafana/dashboards\n\ + \ foldersFromFilesStructure: false\n\n" + kind: ConfigMap + metadata: + name: prov-dash + namespace: trustgraph +- apiVersion: v1 + data: + datasource.yml: "apiVersion: 1\n\nprune: true\n\ndatasources:\n - name: Prometheus\n\ + \ type: prometheus\n access: proxy\n orgId: 1\n # Sets\ + \ a custom UID to reference this\n # data source in other parts of the configuration.\n\ + \ # If not specified, Grafana generates one.\n uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129'\n\ + \n url: http://prometheus:9090\n\n basicAuth: false\n withCredentials:\ + \ false\n isDefault: true\n editable: true\n\n" + kind: ConfigMap + metadata: + name: prov-data + namespace: trustgraph +- apiVersion: v1 + data: + dashboard.json: "{\n \"annotations\": {\n \"list\": [\n {\n \"\ + builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n\ + \ \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n\ + \ \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n\ + \ \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\ + \n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n\ + \ \"graphTooltip\": 0,\n \"id\": 2,\n \"links\": [],\n \"liveNow\": false,\n\ + \ \"panels\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\"\ + ,\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n },\n \ + \ \"fieldConfig\": {\n \"defaults\": {\n \"custom\": {\n \ + \ \"hideFrom\": {\n \"legend\": false,\n \ + \ \"tooltip\": false,\n \"viz\": false\n },\n \ + \ \"scaleDistribution\": {\n \"type\": \"linear\"\n \ + \ }\n }\n },\n \"overrides\": []\n },\n \ + \ \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n\ + \ \"y\": 0\n },\n \"id\": 7,\n \"options\": {\n \ + \ \"calculate\": false,\n \"cellGap\": 1,\n \"color\": {\n \ + \ \"exponent\": 0.5,\n \"fill\": \"dark-orange\",\n \ + \ \"mode\": \"scheme\",\n \"reverse\": false,\n \"scale\"\ + : \"exponential\",\n \"scheme\": \"Oranges\",\n \"steps\"\ + : 64\n },\n \"exemplars\": {\n \"color\": \"rgba(255,0,255,0.7)\"\ + \n },\n \"filterValues\": {\n \"le\": 1e-9\n },\n\ + \ \"legend\": {\n \"show\": true\n },\n \"rowsFrame\"\ + : {\n \"layout\": \"auto\"\n },\n \"tooltip\": {\n \ + \ \"mode\": \"single\",\n \"showColorScale\": false,\n \ + \ \"yHistogram\": false\n },\n \"yAxis\": {\n \"\ + axisPlacement\": \"left\",\n \"reverse\": false\n }\n },\n\ + \ \"pluginVersion\": \"11.1.4\",\n \"targets\": [\n {\n \ + \ \"datasource\": {\n \"type\": \"prometheus\",\n \ + \ \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n },\n \ + \ \"editorMode\": \"code\",\n \"exemplar\": false,\n \"\ + expr\": \"max by(le) (text_completion_duration_bucket)\",\n \"format\"\ + : \"heatmap\",\n \"instant\": false,\n \"legendFormat\": \"\ + 99%\",\n \"range\": true,\n \"refId\": \"A\"\n }\n\ + \ ],\n \"title\": \"LLM latency\",\n \"type\": \"heatmap\"\n\ + \ },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n\ + \ \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n },\n \ + \ \"fieldConfig\": {\n \"defaults\": {\n \"custom\": {\n \ + \ \"hideFrom\": {\n \"legend\": false,\n \"\ + tooltip\": false,\n \"viz\": false\n },\n \ + \ \"scaleDistribution\": {\n \"type\": \"linear\"\n \ + \ }\n }\n },\n \"overrides\": []\n },\n \"\ + gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \ + \ \"y\": 0\n },\n \"id\": 2,\n \"options\": {\n \"\ + calculate\": false,\n \"cellGap\": 5,\n \"cellValues\": {\n \ + \ \"unit\": \"\"\n },\n \"color\": {\n \"exponent\"\ + : 0.5,\n \"fill\": \"dark-orange\",\n \"mode\": \"scheme\"\ + ,\n \"reverse\": false,\n \"scale\": \"exponential\",\n \ + \ \"scheme\": \"Oranges\",\n \"steps\": 64\n },\n \ + \ \"exemplars\": {\n \"color\": \"rgba(255,0,255,0.7)\"\n \ + \ },\n \"filterValues\": {\n \"le\": 1e-9\n },\n \ + \ \"legend\": {\n \"show\": true\n },\n \"rowsFrame\"\ + : {\n \"layout\": \"auto\"\n },\n \"tooltip\": {\n \ + \ \"mode\": \"single\",\n \"showColorScale\": false,\n \ + \ \"yHistogram\": false\n },\n \"yAxis\": {\n \"\ + axisLabel\": \"processing status\",\n \"axisPlacement\": \"left\",\n\ + \ \"reverse\": false\n }\n },\n \"pluginVersion\"\ + : \"11.1.4\",\n \"targets\": [\n {\n \"datasource\": {\n\ + \ \"type\": \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"disableTextWrap\": false,\n \"editorMode\"\ + : \"builder\",\n \"exemplar\": false,\n \"expr\": \"sum by(status)\ + \ (rate(processing_count_total{status!=\\\"success\\\"}[$__rate_interval]))\"\ + ,\n \"format\": \"heatmap\",\n \"fullMetaSearch\": false,\n\ + \ \"includeNullMetadata\": true,\n \"instant\": false,\n \ + \ \"interval\": \"\",\n \"legendFormat\": \"{{status}}\",\n\ + \ \"range\": true,\n \"refId\": \"A\",\n \"useBackend\"\ + : false\n }\n ],\n \"title\": \"Error rate\",\n \"type\"\ + : \"heatmap\"\n },\n {\n \"datasource\": {\n \"type\": \"\ + prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n \ + \ },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\"\ + : {\n \"mode\": \"palette-classic\"\n },\n \"custom\"\ + : {\n \"axisBorderShow\": false,\n \"axisCenteredZero\"\ + : false,\n \"axisColorMode\": \"text\",\n \"axisLabel\"\ + : \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\"\ + : 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n\ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \ + \ \"viz\": false\n },\n \"insertNulls\": false,\n\ + \ \"lineInterpolation\": \"linear\",\n \"lineWidth\":\ + \ 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n\ + \ \"type\": \"linear\"\n },\n \"showPoints\"\ + : \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n\ + \ \"group\": \"A\",\n \"mode\": \"none\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\ + \n }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 9,\n \"w\": 12,\n \"x\": 0,\n \"y\": 8\n\ + \ },\n \"id\": 1,\n \"options\": {\n \"legend\": {\n \ + \ \"calcs\": [],\n \"displayMode\": \"list\",\n \"\ + placement\": \"bottom\",\n \"showLegend\": true\n },\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"\ + none\"\n }\n },\n \"targets\": [\n {\n \"datasource\"\ + : {\n \"type\": \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"editorMode\": \"builder\",\n \"expr\":\ + \ \"rate(request_latency_count[1m])\",\n \"instant\": false,\n \ + \ \"legendFormat\": \"{{instance}}\",\n \"range\": true,\n \ + \ \"refId\": \"A\"\n }\n ],\n \"title\": \"Request rate\"\ + ,\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n\ + \ \"type\": \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"mode\": \"palette-classic\"\n },\n \ + \ \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\"\ + : false,\n \"axisColorMode\": \"text\",\n \"axisLabel\"\ + : \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\"\ + : 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n\ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \ + \ \"viz\": false\n },\n \"insertNulls\": false,\n\ + \ \"lineInterpolation\": \"linear\",\n \"lineWidth\":\ + \ 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n\ + \ \"type\": \"linear\"\n },\n \"showPoints\"\ + : \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n\ + \ \"group\": \"A\",\n \"mode\": \"none\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\ + \n }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 9,\n \"w\": 12,\n \"x\": 12,\n \"y\": 8\n\ + \ },\n \"id\": 5,\n \"options\": {\n \"legend\": {\n \ + \ \"calcs\": [],\n \"displayMode\": \"list\",\n \"\ + placement\": \"bottom\",\n \"showLegend\": true\n },\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"\ + none\"\n }\n },\n \"pluginVersion\": \"10.0.0\",\n \"\ + targets\": [\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"editorMode\": \"builder\",\n \"expr\":\ + \ \"pulsar_msg_backlog\",\n \"instant\": false,\n \"legendFormat\"\ + : \"{{topic}}\",\n \"range\": true,\n \"refId\": \"A\"\n \ + \ }\n ],\n \"title\": \"Pub/sub backlog\",\n \"type\":\ + \ \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"fixedColor\": \"semi-dark-green\",\n \"\ + mode\": \"palette-classic-by-name\"\n },\n \"custom\": {\n\ + \ \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n\ + \ \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n\ + \ \"axisPlacement\": \"auto\",\n \"fillOpacity\": 80,\n\ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \ + \ \"viz\": false\n },\n \"lineWidth\": 1,\n \ + \ \"scaleDistribution\": {\n \"type\": \"linear\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\ + \n }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 7,\n \"w\": 12,\n \"x\": 0,\n \"y\": 17\n\ + \ },\n \"id\": 10,\n \"options\": {\n \"barRadius\": 0,\n\ + \ \"barWidth\": 0.97,\n \"fullHighlight\": false,\n \"\ + groupWidth\": 0.7,\n \"legend\": {\n \"calcs\": [],\n \ + \ \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \ + \ \"showLegend\": true\n },\n \"orientation\": \"auto\",\n\ + \ \"showValue\": \"auto\",\n \"stacking\": \"none\",\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"none\"\ + \n },\n \"xTickLabelRotation\": 0,\n \"xTickLabelSpacing\"\ + : 0\n },\n \"pluginVersion\": \"11.1.4\",\n \"targets\": [\n\ + \ {\n \"datasource\": {\n \"type\": \"prometheus\"\ + ,\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n \ + \ },\n \"disableTextWrap\": false,\n \"editorMode\": \"builder\"\ + ,\n \"exemplar\": false,\n \"expr\": \"max by(le) (chunk_size_bucket)\"\ + ,\n \"format\": \"heatmap\",\n \"fullMetaSearch\": false,\n\ + \ \"includeNullMetadata\": false,\n \"instant\": true,\n \ + \ \"legendFormat\": \"{{le}}\",\n \"range\": false,\n \ + \ \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n\ + \ \"title\": \"Chunk size\",\n \"type\": \"barchart\"\n },\n \ + \ {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"\ + uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n },\n \"fieldConfig\"\ + : {\n \"defaults\": {\n \"color\": {\n \"mode\":\ + \ \"palette-classic\"\n },\n \"custom\": {\n \"\ + axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \ + \ \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \ + \ \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \ + \ \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \"\ + viz\": false\n },\n \"insertNulls\": false,\n \ + \ \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \ + \ \"pointSize\": 5,\n \"scaleDistribution\": {\n \ + \ \"type\": \"linear\"\n },\n \"showPoints\": \"\ + auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \ + \ \"group\": \"A\",\n \"mode\": \"none\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n\ + \ }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 7,\n \"w\": 12,\n \"x\": 12,\n \"y\": 17\n\ + \ },\n \"id\": 11,\n \"options\": {\n \"legend\": {\n\ + \ \"calcs\": [],\n \"displayMode\": \"list\",\n \"\ + placement\": \"bottom\",\n \"showLegend\": true\n },\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"\ + none\"\n }\n },\n \"pluginVersion\": \"11.1.4\",\n \"\ + targets\": [\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"disableTextWrap\": false,\n \"editorMode\"\ + : \"builder\",\n \"exemplar\": false,\n \"expr\": \"increase(processing_count_total{status!=\\\ + \"success\\\"}[$__rate_interval])\",\n \"format\": \"time_series\"\ + ,\n \"fullMetaSearch\": false,\n \"includeNullMetadata\":\ + \ true,\n \"instant\": false,\n \"legendFormat\": \"{{instance}}\"\ + ,\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\"\ + : false\n }\n ],\n \"title\": \"Errors\",\n \"type\":\ + \ \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"fixedColor\": \"light-blue\",\n \"mode\"\ + : \"palette-classic\"\n },\n \"custom\": {\n \"\ + axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \ + \ \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \ + \ \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \ + \ \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \"\ + viz\": false\n },\n \"insertNulls\": false,\n \ + \ \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \ + \ \"pointSize\": 5,\n \"scaleDistribution\": {\n \ + \ \"type\": \"linear\"\n },\n \"showPoints\": \"\ + auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \ + \ \"group\": \"A\",\n \"mode\": \"none\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n\ + \ }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 24\n\ + \ },\n \"id\": 12,\n \"options\": {\n \"legend\": {\n\ + \ \"calcs\": [],\n \"displayMode\": \"list\",\n \"\ + placement\": \"bottom\",\n \"showLegend\": true\n },\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"\ + none\"\n }\n },\n \"pluginVersion\": \"11.1.4\",\n \"\ + targets\": [\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"disableTextWrap\": false,\n \"editorMode\"\ + : \"builder\",\n \"expr\": \"rate(process_cpu_seconds_total[$__rate_interval])\"\ + ,\n \"fullMetaSearch\": false,\n \"includeNullMetadata\":\ + \ true,\n \"instant\": false,\n \"legendFormat\": \"{{instance}}\"\ + ,\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\"\ + : false\n }\n ],\n \"title\": \"CPU\",\n \"type\": \"\ + timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\"\ + ,\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\n },\n \ + \ \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \ + \ \"mode\": \"palette-classic\"\n },\n \"custom\"\ + : {\n \"axisBorderShow\": false,\n \"axisCenteredZero\"\ + : false,\n \"axisColorMode\": \"text\",\n \"axisLabel\"\ + : \"GB\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\"\ + : 0,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n\ + \ \"gradientMode\": \"none\",\n \"hideFrom\": {\n \ + \ \"legend\": false,\n \"tooltip\": false,\n \ + \ \"viz\": false\n },\n \"insertNulls\": false,\n\ + \ \"lineInterpolation\": \"linear\",\n \"lineWidth\":\ + \ 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n\ + \ \"type\": \"linear\"\n },\n \"showPoints\"\ + : \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n\ + \ \"group\": \"A\",\n \"mode\": \"none\"\n \ + \ },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\ + \n }\n },\n \"mappings\": [],\n \"thresholds\"\ + : {\n \"mode\": \"absolute\",\n \"steps\": [\n \ + \ {\n \"color\": \"green\",\n \"value\"\ + : null\n },\n {\n \"color\": \"red\"\ + ,\n \"value\": 80\n }\n ]\n \ + \ }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n\ + \ \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 24\n\ + \ },\n \"id\": 13,\n \"options\": {\n \"legend\": {\n\ + \ \"calcs\": [],\n \"displayMode\": \"list\",\n \"\ + placement\": \"bottom\",\n \"showLegend\": true\n },\n \ + \ \"tooltip\": {\n \"mode\": \"single\",\n \"sort\": \"\ + none\"\n }\n },\n \"targets\": [\n {\n \"datasource\"\ + : {\n \"type\": \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"disableTextWrap\": false,\n \"editorMode\"\ + : \"builder\",\n \"expr\": \"process_resident_memory_bytes / 1073741824\"\ + ,\n \"fullMetaSearch\": false,\n \"includeNullMetadata\":\ + \ true,\n \"instant\": false,\n \"legendFormat\": \"{{instance}}\"\ + ,\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\"\ + : false\n }\n ],\n \"title\": \"Memory\",\n \"type\":\ + \ \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"fieldConfig\": {\n \"defaults\": {\n \"\ + color\": {\n \"mode\": \"thresholds\"\n },\n \"\ + custom\": {\n \"align\": \"auto\",\n \"cellOptions\":\ + \ {\n \"type\": \"auto\"\n },\n \"filterable\"\ + : false,\n \"inspect\": false\n },\n \"mappings\"\ + : [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \ + \ \"steps\": [\n {\n \"color\": \"green\"\ + ,\n \"value\": null\n },\n {\n \ + \ \"color\": \"red\",\n \"value\": 80\n \ + \ }\n ]\n }\n },\n \"overrides\": []\n\ + \ },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 12,\n \ + \ \"x\": 0,\n \"y\": 32\n },\n \"id\": 14,\n \"options\"\ + : {\n \"cellHeight\": \"sm\",\n \"footer\": {\n \"countRows\"\ + : false,\n \"fields\": \"\",\n \"reducer\": [\n \ + \ \"sum\"\n ],\n \"show\": false\n },\n \"\ + showHeader\": true\n },\n \"pluginVersion\": \"11.1.4\",\n \"\ + targets\": [\n {\n \"datasource\": {\n \"type\":\ + \ \"prometheus\",\n \"uid\": \"f6b18033-5918-4e05-a1ca-4cb30343b129\"\ + \n },\n \"disableTextWrap\": false,\n \"editorMode\"\ + : \"builder\",\n \"exemplar\": false,\n \"expr\": \"last_over_time(params_info[$__interval])\"\ + ,\n \"format\": \"table\",\n \"fullMetaSearch\": false,\n\ + \ \"includeNullMetadata\": true,\n \"instant\": true,\n \ + \ \"legendFormat\": \"__auto\",\n \"range\": false,\n \ + \ \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n\ + \ \"title\": \"Model parameters\",\n \"transformations\": [\n \ + \ {\n \"id\": \"filterFieldsByName\",\n \"options\": {\n\ + \ \"include\": {\n \"names\": [\n \"\ + instance\",\n \"model\"\n ]\n }\n \ + \ }\n },\n {\n \"id\": \"filterByValue\",\n \ + \ \"options\": {\n \"filters\": [\n {\n \ + \ \"config\": {\n \"id\": \"equal\",\n \ + \ \"options\": {\n \"value\": \"\"\n \ + \ }\n },\n \"fieldName\": \"model\"\n \ + \ }\n ],\n \"match\": \"all\",\n \"\ + type\": \"exclude\"\n }\n }\n ],\n \"type\": \"table\"\ + \n }\n ],\n \"refresh\": \"5s\",\n \"schemaVersion\": 39,\n \"tags\"\ + : [],\n \"templating\": {\n \"list\": []\n },\n \"time\": {\n \"from\"\ + : \"now-5m\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\"\ + : \"\",\n \"title\": \"Overview\",\n \"uid\": \"b5c8abf8-fe79-496b-b028-10bde917d1f0\"\ + ,\n \"version\": 3,\n \"weekStart\": \"\"\n}\n" + kind: ConfigMap + metadata: + name: dashboards + namespace: trustgraph +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: grafana + name: grafana + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + containers: + - env: + - name: GF_ORG_NAME + value: trustgraph.ai + image: docker.io/grafana/grafana:11.1.4 + name: grafana + ports: + - containerPort: 3000 + hostPort: 3000 + resources: + limits: + cpu: '1.0' + memory: 256M + requests: + cpu: '0.5' + memory: 256M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /var/lib/grafana + name: grafana-storage + - mountPath: /etc/grafana/provisioning/dashboards/ + name: prov-dash + - mountPath: /etc/grafana/provisioning/datasources/ + name: prov-data + - mountPath: /var/lib/grafana/dashboards/ + name: dashboards + volumes: + - name: grafana-storage + persistentVolumeClaim: + claimName: grafana-storage + - configMap: + name: prov-dash + name: prov-dash + - configMap: + name: prov-data + name: prov-data + - configMap: + name: dashboards + name: dashboards +- apiVersion: v1 + kind: Service + metadata: + name: grafana + namespace: trustgraph + spec: + ports: + - name: http + port: 3000 + targetPort: 3000 + selector: + app: grafana + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: graph-rag + name: graph-rag + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: graph-rag + template: + metadata: + labels: + app: graph-rag + spec: + containers: + - command: + - graph-rag + - -p + - pulsar://pulsar:6650 + - --prompt-request-queue + - non-persistent://tg/request/prompt-rag + - --prompt-response-queue + - non-persistent://tg/response/prompt-rag-response + - --entity-limit + - '50' + - --triple-limit + - '30' + - --max-subgraph-size + - '3000' + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: graph-rag + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: graph-rag + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: graph-rag + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-definitions + name: kg-extract-definitions + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-definitions + template: + metadata: + labels: + app: kg-extract-definitions + spec: + containers: + - command: + - kg-extract-definitions + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-definitions + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: kg-extract-definitions + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: kg-extract-definitions + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-relationships + name: kg-extract-relationships + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-relationships + template: + metadata: + labels: + app: kg-extract-relationships + spec: + containers: + - command: + - kg-extract-relationships + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-relationships + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: kg-extract-relationships + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: kg-extract-relationships + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: PersistentVolume + metadata: + name: neo4j + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 20G + hostPath: + path: /data/pv-neo4j + persistentVolumeReclaimPolicy: Delete +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: neo4j + namespace: trustgraph + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20G +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: neo4j + name: neo4j + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: neo4j + template: + metadata: + labels: + app: neo4j + spec: + containers: + - env: + - name: NEO4J_AUTH + value: neo4j/password + image: docker.io/neo4j:5.22.0-community-bullseye + name: neo4j + ports: + - containerPort: 7474 + hostPort: 7474 + - containerPort: 7687 + hostPort: 7687 + resources: + limits: + cpu: '1.0' + memory: 768M + requests: + cpu: '0.5' + memory: 768M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /data + name: neo4j + volumes: + - name: neo4j + persistentVolumeClaim: + claimName: neo4j +- apiVersion: v1 + kind: Service + metadata: + name: neo4j + namespace: trustgraph + spec: + ports: + - name: api + port: 7474 + targetPort: 7474 + - name: api2 + port: 7687 + targetPort: 7687 + selector: + app: neo4j + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: pdf-decoder + name: pdf-decoder + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: pdf-decoder + template: + metadata: + labels: + app: pdf-decoder + spec: + containers: + - command: + - pdf-decoder + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: pdf-decoder + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: pdf-decoder + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: pdf-decoder + type: LoadBalancer +- apiVersion: v1 + data: + prometheus.yml: "global:\n\n scrape_interval: 15s # By default, scrape targets\ + \ every 15 seconds.\n\n # Attach these labels to any time series or alerts\ + \ when communicating with\n # external systems (federation, remote storage,\ + \ Alertmanager).\n external_labels:\n monitor: 'trustgraph'\n\n# A scrape\ + \ configuration containing exactly one endpoint to scrape:\n# Here it's Prometheus\ + \ itself.\nscrape_configs:\n\n # The job name is added as a label `job=`\ + \ to any timeseries\n # scraped from this config.\n\n - job_name: 'trustgraph'\n\ + \n # Override the global default and scrape targets from this job every\n\ + \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ + \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ + \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" + kind: ConfigMap + metadata: + name: prometheus-cfg + namespace: trustgraph +- apiVersion: v1 + kind: PersistentVolume + metadata: + name: prometheus-data + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 20G + hostPath: + path: /data/pv-prometheus-data + persistentVolumeReclaimPolicy: Delete +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: prometheus-data + namespace: trustgraph + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20G +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: prometheus + name: prometheus + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + containers: + - image: docker.io/prom/prometheus:v2.53.2 + name: prometheus + ports: + - containerPort: 9090 + hostPort: 9090 + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /etc/prometheus/ + name: prometheus-cfg + - mountPath: /prometheus + name: prometheus-data + volumes: + - configMap: + name: prometheus-cfg + name: prometheus-cfg + - name: prometheus-data + persistentVolumeClaim: + claimName: prometheus-data +- apiVersion: v1 + kind: Service + metadata: + name: prometheus + namespace: trustgraph + spec: + ports: + - name: http + port: 9090 + targetPort: 9090 + selector: + app: prometheus + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: prompt + name: prompt + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: prompt + template: + metadata: + labels: + app: prompt + spec: + containers: + - command: + - prompt-generic + - -p + - pulsar://pulsar:6650 + - --text-completion-request-queue + - non-persistent://tg/request/text-completion + - --text-completion-response-queue + - non-persistent://tg/response/text-completion-response + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: prompt + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: prompt + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: prompt + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: prompt-rag + name: prompt-rag + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: prompt-rag + template: + metadata: + labels: + app: prompt-rag + spec: + containers: + - command: + - prompt-generic + - -p + - pulsar://pulsar:6650 + - -i + - non-persistent://tg/request/prompt-rag + - -o + - non-persistent://tg/response/prompt-rag-response + - --text-completion-request-queue + - non-persistent://tg/request/text-completion-rag + - --text-completion-response-queue + - non-persistent://tg/response/text-completion-rag-response + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: prompt-rag + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: prompt-rag + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: prompt-rag + type: LoadBalancer +- apiVersion: v1 + kind: PersistentVolume + metadata: + name: pulsar-data + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 20G + hostPath: + path: /data/pv-pulsar-data + persistentVolumeReclaimPolicy: Delete +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: pulsar-data + namespace: trustgraph + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20G +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: pulsar + name: pulsar + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: pulsar + template: + metadata: + labels: + app: pulsar + spec: + containers: + - command: + - bin/pulsar + - standalone + env: + - name: PULSAR_MEM + value: -Xms600M -Xmx600M + image: docker.io/apachepulsar/pulsar:3.3.1 + name: pulsar + ports: + - containerPort: 6650 + hostPort: 6650 + - containerPort: 8080 + hostPort: 8080 + resources: + limits: + cpu: '2.0' + memory: 1500M + requests: + cpu: '1.0' + memory: 1500M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /pulsar/data + name: pulsar-data + volumes: + - name: pulsar-data + persistentVolumeClaim: + claimName: pulsar-data +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: init-pulsar + name: init-pulsar + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: init-pulsar + template: + metadata: + labels: + app: init-pulsar + spec: + containers: + - command: + - sh + - -c + - while true; do pulsar-admin --admin-url http://pulsar:8080 tenants create + tg ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/flow + ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/request + ; pulsar-admin --admin-url http://pulsar:8080 namespaces create tg/response + ; pulsar-admin --admin-url http://pulsar:8080 namespaces set-retention + --size -1 --time 3m tg/response; sleep 20; done + image: docker.io/apachepulsar/pulsar:3.3.1 + name: init-pulsar + resources: + limits: + cpu: '1' + memory: 400M + requests: + cpu: '0.1' + memory: 400M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: pulsar + namespace: trustgraph + spec: + ports: + - name: bookie + port: 6650 + targetPort: 6650 + - name: http + port: 8080 + targetPort: 8080 + selector: + app: pulsar + type: LoadBalancer +- apiVersion: v1 + kind: PersistentVolume + metadata: + name: qdrant + spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 20G + hostPath: + path: /data/pv-qdrant + persistentVolumeReclaimPolicy: Delete +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: qdrant + namespace: trustgraph + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20G +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: qdrant + name: qdrant + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: qdrant + template: + metadata: + labels: + app: qdrant + spec: + containers: + - image: docker.io/qdrant/qdrant:v1.11.1 + name: qdrant + ports: + - containerPort: 6333 + hostPort: 6333 + - containerPort: 6334 + hostPort: 6334 + resources: + limits: + cpu: '1.0' + memory: 256M + requests: + cpu: '0.5' + memory: 256M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumeMounts: + - mountPath: /qdrant/storage + name: qdrant + volumes: + - name: qdrant + persistentVolumeClaim: + claimName: qdrant +- apiVersion: v1 + kind: Service + metadata: + name: qdrant + namespace: trustgraph + spec: + ports: + - name: api + port: 6333 + targetPort: 6333 + - name: api2 + port: 6334 + targetPort: 6334 + selector: + app: qdrant + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: query-doc-embeddings + name: query-doc-embeddings + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: query-doc-embeddings + template: + metadata: + labels: + app: query-doc-embeddings + spec: + containers: + - command: + - de-query-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: query-doc-embeddings + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: query-doc-embeddings + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: query-doc-embeddings + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: query-graph-embeddings + name: query-graph-embeddings + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: query-graph-embeddings + template: + metadata: + labels: + app: query-graph-embeddings + spec: + containers: + - command: + - ge-query-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: query-graph-embeddings + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: query-graph-embeddings + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: query-graph-embeddings + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: query-triples + name: query-triples + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: query-triples + template: + metadata: + labels: + app: query-triples + spec: + containers: + - command: + - triples-query-neo4j + - -p + - pulsar://pulsar:6650 + - -g + - bolt://neo4j:7687 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: query-triples + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: query-triples + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: query-triples + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: store-doc-embeddings + name: store-doc-embeddings + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: store-doc-embeddings + template: + metadata: + labels: + app: store-doc-embeddings + spec: + containers: + - command: + - de-write-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: store-doc-embeddings + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: store-doc-embeddings + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: store-doc-embeddings + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: store-graph-embeddings + name: store-graph-embeddings + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: store-graph-embeddings + template: + metadata: + labels: + app: store-graph-embeddings + spec: + containers: + - command: + - ge-write-qdrant + - -p + - pulsar://pulsar:6650 + - -t + - http://qdrant:6333 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: store-graph-embeddings + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: store-graph-embeddings + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: store-graph-embeddings + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: store-triples + name: store-triples + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: store-triples + template: + metadata: + labels: + app: store-triples + spec: + containers: + - command: + - triples-write-neo4j + - -p + - pulsar://pulsar:6650 + - -g + - bolt://neo4j:7687 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: store-triples + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: store-triples + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8080 + targetPort: 8080 + selector: + app: store-triples + type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: text-completion + name: text-completion + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: text-completion + template: + metadata: + labels: + app: text-completion + spec: + containers: + - command: + - text-completion-llamafile + - -p + - pulsar://pulsar:6650 + - -m + - LLaMA_CPP + - -r + - ${LLAMAFILE_URL} + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: text-completion + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: text-completion-rag + name: text-completion-rag + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: text-completion-rag + template: + metadata: + labels: + app: text-completion-rag + spec: + containers: + - command: + - text-completion-llamafile + - -p + - pulsar://pulsar:6650 + - -m + - LLaMA_CPP + - -r + - ${LLAMAFILE_URL} + - -i + - non-persistent://tg/request/text-completion-rag + - -o + - non-persistent://tg/response/text-completion-rag-response + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: text-completion-rag + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: vectorize + name: vectorize + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: vectorize + template: + metadata: + labels: + app: vectorize + spec: + containers: + - command: + - embeddings-vectorize + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: vectorize + resources: + limits: + cpu: '1.0' + memory: 512M + requests: + cpu: '0.5' + memory: 512M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] +- apiVersion: v1 + kind: Service + metadata: + name: vectorize + namespace: trustgraph + spec: + ports: + - name: metrics + port: 8000 + targetPort: 8000 + selector: + app: vectorize + type: LoadBalancer +kind: List + diff --git a/deploy/minikube/tg-launch-ollama-cassandra.yaml b/deploy/minikube/tg-launch-ollama-cassandra.yaml index aca769f1..8eba8502 100644 --- a/deploy/minikube/tg-launch-ollama-cassandra.yaml +++ b/deploy/minikube/tg-launch-ollama-cassandra.yaml @@ -110,7 +110,7 @@ items: - '250' - --chunk-overlap - '15' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -160,7 +160,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -669,7 +669,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -717,7 +717,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -765,7 +765,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -791,6 +791,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: apps/v1 kind: Deployment metadata: @@ -813,7 +848,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -852,11 +887,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -968,7 +1004,7 @@ items: - non-persistent://tg/request/text-completion - --text-completion-response-queue - non-persistent://tg/response/text-completion-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1024,7 +1060,7 @@ items: - non-persistent://tg/request/text-completion-rag - --text-completion-response-queue - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1282,7 +1318,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1332,7 +1368,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1382,7 +1418,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1432,7 +1468,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1482,7 +1518,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1532,7 +1568,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1584,7 +1620,7 @@ items: - gemma2:9b - -r - ${OLLAMA_HOST} - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1640,7 +1676,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1688,7 +1724,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-ollama-neo4j.yaml b/deploy/minikube/tg-launch-ollama-neo4j.yaml index bb2c70b1..bd4e6f25 100644 --- a/deploy/minikube/tg-launch-ollama-neo4j.yaml +++ b/deploy/minikube/tg-launch-ollama-neo4j.yaml @@ -31,7 +31,7 @@ items: - '250' - --chunk-overlap - '15' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -81,7 +81,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -590,7 +590,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -638,7 +638,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -686,7 +686,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -712,6 +712,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: v1 kind: PersistentVolume metadata: @@ -818,7 +853,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -857,11 +892,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -973,7 +1009,7 @@ items: - non-persistent://tg/request/text-completion - --text-completion-response-queue - non-persistent://tg/response/text-completion-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1029,7 +1065,7 @@ items: - non-persistent://tg/request/text-completion-rag - --text-completion-response-queue - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1287,7 +1323,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1337,7 +1373,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1387,7 +1423,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1437,7 +1473,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1487,7 +1523,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1537,7 +1573,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1589,7 +1625,7 @@ items: - gemma2:9b - -r - ${OLLAMA_HOST} - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1645,7 +1681,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1693,7 +1729,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-openai-cassandra.yaml b/deploy/minikube/tg-launch-openai-cassandra.yaml index 7791580f..ea624801 100644 --- a/deploy/minikube/tg-launch-openai-cassandra.yaml +++ b/deploy/minikube/tg-launch-openai-cassandra.yaml @@ -110,7 +110,7 @@ items: - '250' - --chunk-overlap - '15' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -160,7 +160,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -669,7 +669,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -717,7 +717,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -765,7 +765,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -791,6 +791,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: apps/v1 kind: Deployment metadata: @@ -813,7 +848,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -852,11 +887,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1043,6 +1079,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1113,7 +1161,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1244,6 +1292,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1314,7 +1374,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1572,7 +1632,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1622,7 +1682,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1672,7 +1732,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1722,7 +1782,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1772,7 +1832,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1822,7 +1882,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1878,7 +1938,7 @@ items: - '0' - -m - GPT-3.5-Turbo - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1938,7 +1998,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1986,7 +2046,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-openai-neo4j.yaml b/deploy/minikube/tg-launch-openai-neo4j.yaml index cc96c4c5..de7a83bb 100644 --- a/deploy/minikube/tg-launch-openai-neo4j.yaml +++ b/deploy/minikube/tg-launch-openai-neo4j.yaml @@ -31,7 +31,7 @@ items: - '250' - --chunk-overlap - '15' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -81,7 +81,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -590,7 +590,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -638,7 +638,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -686,7 +686,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -712,6 +712,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: v1 kind: PersistentVolume metadata: @@ -818,7 +853,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -857,11 +892,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1048,6 +1084,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1118,7 +1166,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1249,6 +1297,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1319,7 +1379,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1577,7 +1637,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1627,7 +1687,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1677,7 +1737,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1727,7 +1787,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1777,7 +1837,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1827,7 +1887,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1883,7 +1943,7 @@ items: - '0' - -m - GPT-3.5-Turbo - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1943,7 +2003,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -1991,7 +2051,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-vertexai-cassandra.yaml b/deploy/minikube/tg-launch-vertexai-cassandra.yaml index d2ec6f43..a7bcb759 100644 --- a/deploy/minikube/tg-launch-vertexai-cassandra.yaml +++ b/deploy/minikube/tg-launch-vertexai-cassandra.yaml @@ -110,7 +110,7 @@ items: - '250' - --chunk-overlap - '15' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -160,7 +160,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -669,7 +669,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -717,7 +717,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -765,7 +765,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -791,6 +791,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: apps/v1 kind: Deployment metadata: @@ -813,7 +848,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -852,11 +887,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1043,6 +1079,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1113,7 +1161,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1244,6 +1292,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1314,7 +1374,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1572,7 +1632,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1622,7 +1682,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1672,7 +1732,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1722,7 +1782,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1772,7 +1832,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1822,7 +1882,7 @@ items: - pulsar://pulsar:6650 - -g - cassandra - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1850,7 +1910,7 @@ items: type: LoadBalancer - apiVersion: v1 data: - private.json: UFJJVkFURSBLRVkgR09FUyBIRVJFCg== + private.json: ewogICJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsCiAgInByb2plY3RfaWQiOiAia2FsbnRlcmEtZGVtbyIsCiAgInByaXZhdGVfa2V5X2lkIjogIjY5MjgwNTI5ODAyMjI3YmJiZjk1YTRjNjdjZjZhZjg3MGM3MTE4ZjIiLAogICJwcml2YXRlX2tleSI6ICItLS0tLUJFR0lOIFBSSVZBVEUgS0VZLS0tLS1cbk1JSUV2d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktrd2dnU2xBZ0VBQW9JQkFRRFV4dTBtYkdXVHhEOFFcbkVsd2J0YVZEbER2UlA4MkhGb3QyaUJVejRoQ0hpS04vVkpRM2VGNnJXa05iSnI4endia21XTzMydkxubU4vNnNcbk5qeEtFWTJmYUwrbkhydkZXa2xRQ0VMWlM2V0tmT1NrMTgxUXkyNE9Hd2E2TTFPdE85SXpWWisxNXFRN09Ib2hcblQ5MFdyOTVMYVcrTVlqMTAvc1RpeG5tRzFrMitEOGhUNjhCNnhrZHlRc0VSaThHbURaN2JzNWU5UEhMQ0NKU1lcbi9lTXcxSnU1V3YvM0VWZGVvMENTNjE1cXl1enRuS3NRQ3RjbWJFY1oxcDI4cVdqMXh3ckw4Wk1QaU1DZm1ldzlcbkF4STAyNkJmWXI5aFR5TGRqdWNSQ3p4NkNOZml5TUZxaXROMDJPZFo5MjNZSmZWckJIU2h3MDRKSlhIV0FqNkhcbkExTXJNZU12QWdNQkFBRUNnZ0VBYVg1N1kvcUlYandGZkVWdUQzci9Ca04vYU42Mld3ek5kcVkveE9xSGdYczRcbmZOQUZjSDhvaU5LRzVMR1FTT3Y2alUyNEpnWXFNVDkvOEFndjJoVmIzelpERjJ2QWN6ZXhjNUcwK3lGcllQOU5cbktZR3BWUzZYVVQwTEY5U00zeTN5cU9zQjlXRDEranFsS3RENjc5MmdYRWxaT2dRWExtT0dZRjFCTVo0ZEMyYkdcblg3UVRZbmUwUnlWWGw0R3BMRlZUNU1XYmpmZUJidUxYdGtNTEZGVjFQbnMwY2NyMUFwcnJDOVNlckp5TjRJL3ZcblpWTkZIZFArM0R2U0M5NlMzYU5OOXhTWG5zSExwby9nc3RJeW5XbWFBdXBpT3hNSm4wVm1MNlBkRStzT01GdWJcbmpyMnhlTGMyUGNQdDg2RzZJVlBjdkt6QURBWlE2SHhtNUp1ZEwvVDNHUUtCZ1FEcXBRbEMrR2Jvbm1aUkZzcDRcbjkwQlZsV2ZpMjhHM1RibTJRclYzMjgvUEtEQnNlRTZ1U25iUmp6YkxrcUJYNFYyOVdkMFBsODJJdmhLV2Vvc0dcbmpZNWpKeHlvSTFSSy9TbGtPUGJwYWlKNmsyZEhNUTRJai80UUFBS0pLL2czbEJIMXl5djhGWWFNay9lMkhUUWFcbkl4aHJwRWJUbmJkME8xQ2RZb2QrYmU2M3pRS0JnUURvSkdZUUFnZDBkS0NYN3VWTXMzSkRReTRDYk80MnduQXNcbnNtemU5ckVSaFc1M2dKL2N0Zm96eXZsYm5Rc2FjSmpuR1hPNjlWczk5WExZUjg5Zkx0OGFDZFRMUVhqUnIzMkRcbllyTzU3VkVKUmFvYkYxTUJYMk1WMmRnSFRQWDgzTHhUclBtUlhocGVsRGh1STdlWWxvYnRvNlovUUVLRVZvNmZcbkxEdDIzRzNTNndLQmdRRFJjR2Y0NkFIckJsL3JaejFvaGNGZDR0THNraHl4d2VYUEFuSTV2dkVaUmhVYTdMUEdcbkxGOHNJL25UOFZlL3VVQlRvQitLS0lSajFzeCtsbmo3YTJTNXFoS1JIWi9DbzMvTG5uRDkzU2RUWTdqZXViRitcbkFSRFVOVDRPTkRvRldkeENHWXgwSUw2TlFoSHI4dHNyclVIQjZzcnhYaWgremViaDVDMXVzeTlOelFLQmdRRFBcbmRlQUVoRnUzTnBkaTBLdjVyUkQ3amFrcm44QVFEaHFEeDMxR2tYN1RJc1VEWlY3NHQyVUFMbkJFOElkQXZNblpcbkZrVDRHMWZNZW5PRldEKzZWOEFvbkFxaEQwVHpWL3M2bE5YNzNsdWlockNyLzE4K0Jjb2VlQTM1RUUxeUNxNy9cbnRNRmdxWkcxWG85RDF3VzRCeVpVY0FGWERjMmtTVnF5WTZ4TTRGZ0J1d0tCZ1FEQ1AxclF4K2ZoNWdrY1BNUHBcbitYZS9oMDR1SmRSdmpaUkE0MFo3aHViRVFReFNEeWtkMHZSRXFQVlhNS1p1OTFkb0NwRGZHd051dlFkbHY5ZzRcbmc2NDFxRkNXd2ZRb0VZZTBTWUhJVUY5NlRwazVpeGdmUlhYVHRxNGE2c0NvMGV2NG51aFh4Vk82NFRyY0E1YUVcbnB1dWMvYzVTTlh1M0plMWgwK1Jzcm1lWkh3PT1cbi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS1cbiIsCiAgImNsaWVudF9lbWFpbCI6ICJtYXJrLWRldkBrYWxudGVyYS1kZW1vLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAiY2xpZW50X2lkIjogIjExNTExODU2MTEwMTQ1MDM0MDQ0MCIsCiAgImF1dGhfdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi9hdXRoIiwKICAidG9rZW5fdXJpIjogImh0dHBzOi8vb2F1dGgyLmdvb2dsZWFwaXMuY29tL3Rva2VuIiwKICAiYXV0aF9wcm92aWRlcl94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL29hdXRoMi92MS9jZXJ0cyIsCiAgImNsaWVudF94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3JvYm90L3YxL21ldGFkYXRhL3g1MDkvbWFyay1kZXYlNDBrYWxudGVyYS1kZW1vLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAidW5pdmVyc2VfZG9tYWluIjogImdvb2dsZWFwaXMuY29tIgp9Cg== kind: Secret metadata: name: vertexai-creds @@ -1887,7 +1947,7 @@ items: - '0' - -m - gemini-1.0-pro-001 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1921,7 +1981,7 @@ items: type: LoadBalancer - apiVersion: v1 data: - private.json: UFJJVkFURSBLRVkgR09FUyBIRVJFCg== + private.json: ewogICJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsCiAgInByb2plY3RfaWQiOiAia2FsbnRlcmEtZGVtbyIsCiAgInByaXZhdGVfa2V5X2lkIjogIjY5MjgwNTI5ODAyMjI3YmJiZjk1YTRjNjdjZjZhZjg3MGM3MTE4ZjIiLAogICJwcml2YXRlX2tleSI6ICItLS0tLUJFR0lOIFBSSVZBVEUgS0VZLS0tLS1cbk1JSUV2d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktrd2dnU2xBZ0VBQW9JQkFRRFV4dTBtYkdXVHhEOFFcbkVsd2J0YVZEbER2UlA4MkhGb3QyaUJVejRoQ0hpS04vVkpRM2VGNnJXa05iSnI4endia21XTzMydkxubU4vNnNcbk5qeEtFWTJmYUwrbkhydkZXa2xRQ0VMWlM2V0tmT1NrMTgxUXkyNE9Hd2E2TTFPdE85SXpWWisxNXFRN09Ib2hcblQ5MFdyOTVMYVcrTVlqMTAvc1RpeG5tRzFrMitEOGhUNjhCNnhrZHlRc0VSaThHbURaN2JzNWU5UEhMQ0NKU1lcbi9lTXcxSnU1V3YvM0VWZGVvMENTNjE1cXl1enRuS3NRQ3RjbWJFY1oxcDI4cVdqMXh3ckw4Wk1QaU1DZm1ldzlcbkF4STAyNkJmWXI5aFR5TGRqdWNSQ3p4NkNOZml5TUZxaXROMDJPZFo5MjNZSmZWckJIU2h3MDRKSlhIV0FqNkhcbkExTXJNZU12QWdNQkFBRUNnZ0VBYVg1N1kvcUlYandGZkVWdUQzci9Ca04vYU42Mld3ek5kcVkveE9xSGdYczRcbmZOQUZjSDhvaU5LRzVMR1FTT3Y2alUyNEpnWXFNVDkvOEFndjJoVmIzelpERjJ2QWN6ZXhjNUcwK3lGcllQOU5cbktZR3BWUzZYVVQwTEY5U00zeTN5cU9zQjlXRDEranFsS3RENjc5MmdYRWxaT2dRWExtT0dZRjFCTVo0ZEMyYkdcblg3UVRZbmUwUnlWWGw0R3BMRlZUNU1XYmpmZUJidUxYdGtNTEZGVjFQbnMwY2NyMUFwcnJDOVNlckp5TjRJL3ZcblpWTkZIZFArM0R2U0M5NlMzYU5OOXhTWG5zSExwby9nc3RJeW5XbWFBdXBpT3hNSm4wVm1MNlBkRStzT01GdWJcbmpyMnhlTGMyUGNQdDg2RzZJVlBjdkt6QURBWlE2SHhtNUp1ZEwvVDNHUUtCZ1FEcXBRbEMrR2Jvbm1aUkZzcDRcbjkwQlZsV2ZpMjhHM1RibTJRclYzMjgvUEtEQnNlRTZ1U25iUmp6YkxrcUJYNFYyOVdkMFBsODJJdmhLV2Vvc0dcbmpZNWpKeHlvSTFSSy9TbGtPUGJwYWlKNmsyZEhNUTRJai80UUFBS0pLL2czbEJIMXl5djhGWWFNay9lMkhUUWFcbkl4aHJwRWJUbmJkME8xQ2RZb2QrYmU2M3pRS0JnUURvSkdZUUFnZDBkS0NYN3VWTXMzSkRReTRDYk80MnduQXNcbnNtemU5ckVSaFc1M2dKL2N0Zm96eXZsYm5Rc2FjSmpuR1hPNjlWczk5WExZUjg5Zkx0OGFDZFRMUVhqUnIzMkRcbllyTzU3VkVKUmFvYkYxTUJYMk1WMmRnSFRQWDgzTHhUclBtUlhocGVsRGh1STdlWWxvYnRvNlovUUVLRVZvNmZcbkxEdDIzRzNTNndLQmdRRFJjR2Y0NkFIckJsL3JaejFvaGNGZDR0THNraHl4d2VYUEFuSTV2dkVaUmhVYTdMUEdcbkxGOHNJL25UOFZlL3VVQlRvQitLS0lSajFzeCtsbmo3YTJTNXFoS1JIWi9DbzMvTG5uRDkzU2RUWTdqZXViRitcbkFSRFVOVDRPTkRvRldkeENHWXgwSUw2TlFoSHI4dHNyclVIQjZzcnhYaWgremViaDVDMXVzeTlOelFLQmdRRFBcbmRlQUVoRnUzTnBkaTBLdjVyUkQ3amFrcm44QVFEaHFEeDMxR2tYN1RJc1VEWlY3NHQyVUFMbkJFOElkQXZNblpcbkZrVDRHMWZNZW5PRldEKzZWOEFvbkFxaEQwVHpWL3M2bE5YNzNsdWlockNyLzE4K0Jjb2VlQTM1RUUxeUNxNy9cbnRNRmdxWkcxWG85RDF3VzRCeVpVY0FGWERjMmtTVnF5WTZ4TTRGZ0J1d0tCZ1FEQ1AxclF4K2ZoNWdrY1BNUHBcbitYZS9oMDR1SmRSdmpaUkE0MFo3aHViRVFReFNEeWtkMHZSRXFQVlhNS1p1OTFkb0NwRGZHd051dlFkbHY5ZzRcbmc2NDFxRkNXd2ZRb0VZZTBTWUhJVUY5NlRwazVpeGdmUlhYVHRxNGE2c0NvMGV2NG51aFh4Vk82NFRyY0E1YUVcbnB1dWMvYzVTTlh1M0plMWgwK1Jzcm1lWkh3PT1cbi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS1cbiIsCiAgImNsaWVudF9lbWFpbCI6ICJtYXJrLWRldkBrYWxudGVyYS1kZW1vLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAiY2xpZW50X2lkIjogIjExNTExODU2MTEwMTQ1MDM0MDQ0MCIsCiAgImF1dGhfdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi9hdXRoIiwKICAidG9rZW5fdXJpIjogImh0dHBzOi8vb2F1dGgyLmdvb2dsZWFwaXMuY29tL3Rva2VuIiwKICAiYXV0aF9wcm92aWRlcl94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL29hdXRoMi92MS9jZXJ0cyIsCiAgImNsaWVudF94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3JvYm90L3YxL21ldGFkYXRhL3g1MDkvbWFyay1kZXYlNDBrYWxudGVyYS1kZW1vLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAidW5pdmVyc2VfZG9tYWluIjogImdvb2dsZWFwaXMuY29tIgp9Cg== kind: Secret metadata: name: vertexai-creds @@ -1962,7 +2022,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -2016,7 +2076,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: diff --git a/deploy/minikube/tg-launch-vertexai-neo4j.yaml b/deploy/minikube/tg-launch-vertexai-neo4j.yaml index a08b0557..c1b62227 100644 --- a/deploy/minikube/tg-launch-vertexai-neo4j.yaml +++ b/deploy/minikube/tg-launch-vertexai-neo4j.yaml @@ -31,7 +31,7 @@ items: - '250' - --chunk-overlap - '15' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: chunker resources: limits: @@ -81,7 +81,7 @@ items: - pulsar://pulsar:6650 - -m - all-MiniLM-L6-v2 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: embeddings resources: limits: @@ -590,7 +590,7 @@ items: - '30' - --max-subgraph-size - '3000' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: graph-rag resources: limits: @@ -638,7 +638,7 @@ items: - kg-extract-definitions - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-definitions resources: limits: @@ -686,7 +686,7 @@ items: - kg-extract-relationships - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: kg-extract-relationships resources: limits: @@ -712,6 +712,41 @@ items: selector: app: kg-extract-relationships type: LoadBalancer +- apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app: kg-extract-topics + name: kg-extract-topics + namespace: trustgraph + spec: + replicas: 1 + selector: + matchLabels: + app: kg-extract-topics + template: + metadata: + labels: + app: kg-extract-topics + spec: + containers: + - command: + - kg-extract-topics + - -p + - pulsar://pulsar:6650 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 + name: kg-extract-topics + resources: + limits: + cpu: '0.5' + memory: 128M + requests: + cpu: '0.1' + memory: 128M + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: [] - apiVersion: v1 kind: PersistentVolume metadata: @@ -818,7 +853,7 @@ items: - pdf-decoder - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: pdf-decoder resources: limits: @@ -857,11 +892,12 @@ items: \ # 5 seconds.\n scrape_interval: 5s\n\n static_configs:\n - targets:\n\ \ - 'pdf-decoder:8000'\n - 'chunker:8000'\n - 'vectorize:8000'\n\ \ - 'embeddings:8000'\n - 'kg-extract-definitions:8000'\n \ - \ - 'kg-extract-relationships:8000'\n - 'store-graph-embeddings:8000'\n\ - \ - 'store-triples:8000'\n - 'text-completion:8000'\n -\ - \ 'text-completion-rag:8000'\n - 'graph-rag:8000'\n - 'prompt:8000'\n\ - \ - 'prompt-rag:8000'\n - 'query-graph-embeddings:8000'\n \ - \ - 'query-triples:8000'\n - 'pulsar:8080'\n\n" + \ - 'kg-extract-topics:8000'\n - 'kg-extract-relationships:8000'\n\ + \ - 'store-graph-embeddings:8000'\n - 'store-triples:8000'\n \ + \ - 'text-completion:8000'\n - 'text-completion-rag:8000'\n \ + \ - 'graph-rag:8000'\n - 'prompt:8000'\n - 'prompt-rag:8000'\n\ + \ - 'query-graph-embeddings:8000'\n - 'query-triples:8000'\n \ + \ - 'pulsar:8080'\n\n" kind: ConfigMap metadata: name: prometheus-cfg @@ -1048,6 +1084,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1118,7 +1166,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt resources: limits: @@ -1249,6 +1297,18 @@ items: or headers or prefixes. ' + - --topic-template + - "You are a helpful assistant that performs information extraction tasks\ + \ for a provided text.\nRead the provided text. You will identify topics\ + \ and their definitions in JSON.\n\nReading Instructions:\n- Ignore document\ + \ formatting in the provided text.\n- Study the provided text carefully.\n\ + \nHere is the text:\n{text}\n\nResponse Instructions: \n- Do not respond\ + \ with special characters.\n- Return only topics that are concepts and\ + \ unique to the provided text.\n- Respond only with well-formed JSON.\n\ + - The JSON response shall be an array of objects with keys \"topic\" and\ + \ \"definition\". \n- The JSON response shall use the following structure:\n\ + \n```json\n[{{\"topic\": string, \"definition\": string}}]\n```\n\n- Do\ + \ not write any additional text or explanations." - --knowledge-query-template - 'Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only @@ -1319,7 +1379,7 @@ items: explanations. Do not add markdown formatting or headers or prefixes. ' - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: prompt-rag resources: limits: @@ -1577,7 +1637,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-doc-embeddings resources: limits: @@ -1627,7 +1687,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-graph-embeddings resources: limits: @@ -1677,7 +1737,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: query-triples resources: limits: @@ -1727,7 +1787,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-doc-embeddings resources: limits: @@ -1777,7 +1837,7 @@ items: - pulsar://pulsar:6650 - -t - http://qdrant:6333 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-graph-embeddings resources: limits: @@ -1827,7 +1887,7 @@ items: - pulsar://pulsar:6650 - -g - bolt://neo4j:7687 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: store-triples resources: limits: @@ -1855,7 +1915,7 @@ items: type: LoadBalancer - apiVersion: v1 data: - private.json: UFJJVkFURSBLRVkgR09FUyBIRVJFCg== + private.json: ewogICJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsCiAgInByb2plY3RfaWQiOiAia2FsbnRlcmEtZGVtbyIsCiAgInByaXZhdGVfa2V5X2lkIjogIjY5MjgwNTI5ODAyMjI3YmJiZjk1YTRjNjdjZjZhZjg3MGM3MTE4ZjIiLAogICJwcml2YXRlX2tleSI6ICItLS0tLUJFR0lOIFBSSVZBVEUgS0VZLS0tLS1cbk1JSUV2d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktrd2dnU2xBZ0VBQW9JQkFRRFV4dTBtYkdXVHhEOFFcbkVsd2J0YVZEbER2UlA4MkhGb3QyaUJVejRoQ0hpS04vVkpRM2VGNnJXa05iSnI4endia21XTzMydkxubU4vNnNcbk5qeEtFWTJmYUwrbkhydkZXa2xRQ0VMWlM2V0tmT1NrMTgxUXkyNE9Hd2E2TTFPdE85SXpWWisxNXFRN09Ib2hcblQ5MFdyOTVMYVcrTVlqMTAvc1RpeG5tRzFrMitEOGhUNjhCNnhrZHlRc0VSaThHbURaN2JzNWU5UEhMQ0NKU1lcbi9lTXcxSnU1V3YvM0VWZGVvMENTNjE1cXl1enRuS3NRQ3RjbWJFY1oxcDI4cVdqMXh3ckw4Wk1QaU1DZm1ldzlcbkF4STAyNkJmWXI5aFR5TGRqdWNSQ3p4NkNOZml5TUZxaXROMDJPZFo5MjNZSmZWckJIU2h3MDRKSlhIV0FqNkhcbkExTXJNZU12QWdNQkFBRUNnZ0VBYVg1N1kvcUlYandGZkVWdUQzci9Ca04vYU42Mld3ek5kcVkveE9xSGdYczRcbmZOQUZjSDhvaU5LRzVMR1FTT3Y2alUyNEpnWXFNVDkvOEFndjJoVmIzelpERjJ2QWN6ZXhjNUcwK3lGcllQOU5cbktZR3BWUzZYVVQwTEY5U00zeTN5cU9zQjlXRDEranFsS3RENjc5MmdYRWxaT2dRWExtT0dZRjFCTVo0ZEMyYkdcblg3UVRZbmUwUnlWWGw0R3BMRlZUNU1XYmpmZUJidUxYdGtNTEZGVjFQbnMwY2NyMUFwcnJDOVNlckp5TjRJL3ZcblpWTkZIZFArM0R2U0M5NlMzYU5OOXhTWG5zSExwby9nc3RJeW5XbWFBdXBpT3hNSm4wVm1MNlBkRStzT01GdWJcbmpyMnhlTGMyUGNQdDg2RzZJVlBjdkt6QURBWlE2SHhtNUp1ZEwvVDNHUUtCZ1FEcXBRbEMrR2Jvbm1aUkZzcDRcbjkwQlZsV2ZpMjhHM1RibTJRclYzMjgvUEtEQnNlRTZ1U25iUmp6YkxrcUJYNFYyOVdkMFBsODJJdmhLV2Vvc0dcbmpZNWpKeHlvSTFSSy9TbGtPUGJwYWlKNmsyZEhNUTRJai80UUFBS0pLL2czbEJIMXl5djhGWWFNay9lMkhUUWFcbkl4aHJwRWJUbmJkME8xQ2RZb2QrYmU2M3pRS0JnUURvSkdZUUFnZDBkS0NYN3VWTXMzSkRReTRDYk80MnduQXNcbnNtemU5ckVSaFc1M2dKL2N0Zm96eXZsYm5Rc2FjSmpuR1hPNjlWczk5WExZUjg5Zkx0OGFDZFRMUVhqUnIzMkRcbllyTzU3VkVKUmFvYkYxTUJYMk1WMmRnSFRQWDgzTHhUclBtUlhocGVsRGh1STdlWWxvYnRvNlovUUVLRVZvNmZcbkxEdDIzRzNTNndLQmdRRFJjR2Y0NkFIckJsL3JaejFvaGNGZDR0THNraHl4d2VYUEFuSTV2dkVaUmhVYTdMUEdcbkxGOHNJL25UOFZlL3VVQlRvQitLS0lSajFzeCtsbmo3YTJTNXFoS1JIWi9DbzMvTG5uRDkzU2RUWTdqZXViRitcbkFSRFVOVDRPTkRvRldkeENHWXgwSUw2TlFoSHI4dHNyclVIQjZzcnhYaWgremViaDVDMXVzeTlOelFLQmdRRFBcbmRlQUVoRnUzTnBkaTBLdjVyUkQ3amFrcm44QVFEaHFEeDMxR2tYN1RJc1VEWlY3NHQyVUFMbkJFOElkQXZNblpcbkZrVDRHMWZNZW5PRldEKzZWOEFvbkFxaEQwVHpWL3M2bE5YNzNsdWlockNyLzE4K0Jjb2VlQTM1RUUxeUNxNy9cbnRNRmdxWkcxWG85RDF3VzRCeVpVY0FGWERjMmtTVnF5WTZ4TTRGZ0J1d0tCZ1FEQ1AxclF4K2ZoNWdrY1BNUHBcbitYZS9oMDR1SmRSdmpaUkE0MFo3aHViRVFReFNEeWtkMHZSRXFQVlhNS1p1OTFkb0NwRGZHd051dlFkbHY5ZzRcbmc2NDFxRkNXd2ZRb0VZZTBTWUhJVUY5NlRwazVpeGdmUlhYVHRxNGE2c0NvMGV2NG51aFh4Vk82NFRyY0E1YUVcbnB1dWMvYzVTTlh1M0plMWgwK1Jzcm1lWkh3PT1cbi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS1cbiIsCiAgImNsaWVudF9lbWFpbCI6ICJtYXJrLWRldkBrYWxudGVyYS1kZW1vLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAiY2xpZW50X2lkIjogIjExNTExODU2MTEwMTQ1MDM0MDQ0MCIsCiAgImF1dGhfdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi9hdXRoIiwKICAidG9rZW5fdXJpIjogImh0dHBzOi8vb2F1dGgyLmdvb2dsZWFwaXMuY29tL3Rva2VuIiwKICAiYXV0aF9wcm92aWRlcl94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL29hdXRoMi92MS9jZXJ0cyIsCiAgImNsaWVudF94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3JvYm90L3YxL21ldGFkYXRhL3g1MDkvbWFyay1kZXYlNDBrYWxudGVyYS1kZW1vLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAidW5pdmVyc2VfZG9tYWluIjogImdvb2dsZWFwaXMuY29tIgp9Cg== kind: Secret metadata: name: vertexai-creds @@ -1892,7 +1952,7 @@ items: - '0' - -m - gemini-1.0-pro-001 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion resources: limits: @@ -1926,7 +1986,7 @@ items: type: LoadBalancer - apiVersion: v1 data: - private.json: UFJJVkFURSBLRVkgR09FUyBIRVJFCg== + private.json: ewogICJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsCiAgInByb2plY3RfaWQiOiAia2FsbnRlcmEtZGVtbyIsCiAgInByaXZhdGVfa2V5X2lkIjogIjY5MjgwNTI5ODAyMjI3YmJiZjk1YTRjNjdjZjZhZjg3MGM3MTE4ZjIiLAogICJwcml2YXRlX2tleSI6ICItLS0tLUJFR0lOIFBSSVZBVEUgS0VZLS0tLS1cbk1JSUV2d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktrd2dnU2xBZ0VBQW9JQkFRRFV4dTBtYkdXVHhEOFFcbkVsd2J0YVZEbER2UlA4MkhGb3QyaUJVejRoQ0hpS04vVkpRM2VGNnJXa05iSnI4endia21XTzMydkxubU4vNnNcbk5qeEtFWTJmYUwrbkhydkZXa2xRQ0VMWlM2V0tmT1NrMTgxUXkyNE9Hd2E2TTFPdE85SXpWWisxNXFRN09Ib2hcblQ5MFdyOTVMYVcrTVlqMTAvc1RpeG5tRzFrMitEOGhUNjhCNnhrZHlRc0VSaThHbURaN2JzNWU5UEhMQ0NKU1lcbi9lTXcxSnU1V3YvM0VWZGVvMENTNjE1cXl1enRuS3NRQ3RjbWJFY1oxcDI4cVdqMXh3ckw4Wk1QaU1DZm1ldzlcbkF4STAyNkJmWXI5aFR5TGRqdWNSQ3p4NkNOZml5TUZxaXROMDJPZFo5MjNZSmZWckJIU2h3MDRKSlhIV0FqNkhcbkExTXJNZU12QWdNQkFBRUNnZ0VBYVg1N1kvcUlYandGZkVWdUQzci9Ca04vYU42Mld3ek5kcVkveE9xSGdYczRcbmZOQUZjSDhvaU5LRzVMR1FTT3Y2alUyNEpnWXFNVDkvOEFndjJoVmIzelpERjJ2QWN6ZXhjNUcwK3lGcllQOU5cbktZR3BWUzZYVVQwTEY5U00zeTN5cU9zQjlXRDEranFsS3RENjc5MmdYRWxaT2dRWExtT0dZRjFCTVo0ZEMyYkdcblg3UVRZbmUwUnlWWGw0R3BMRlZUNU1XYmpmZUJidUxYdGtNTEZGVjFQbnMwY2NyMUFwcnJDOVNlckp5TjRJL3ZcblpWTkZIZFArM0R2U0M5NlMzYU5OOXhTWG5zSExwby9nc3RJeW5XbWFBdXBpT3hNSm4wVm1MNlBkRStzT01GdWJcbmpyMnhlTGMyUGNQdDg2RzZJVlBjdkt6QURBWlE2SHhtNUp1ZEwvVDNHUUtCZ1FEcXBRbEMrR2Jvbm1aUkZzcDRcbjkwQlZsV2ZpMjhHM1RibTJRclYzMjgvUEtEQnNlRTZ1U25iUmp6YkxrcUJYNFYyOVdkMFBsODJJdmhLV2Vvc0dcbmpZNWpKeHlvSTFSSy9TbGtPUGJwYWlKNmsyZEhNUTRJai80UUFBS0pLL2czbEJIMXl5djhGWWFNay9lMkhUUWFcbkl4aHJwRWJUbmJkME8xQ2RZb2QrYmU2M3pRS0JnUURvSkdZUUFnZDBkS0NYN3VWTXMzSkRReTRDYk80MnduQXNcbnNtemU5ckVSaFc1M2dKL2N0Zm96eXZsYm5Rc2FjSmpuR1hPNjlWczk5WExZUjg5Zkx0OGFDZFRMUVhqUnIzMkRcbllyTzU3VkVKUmFvYkYxTUJYMk1WMmRnSFRQWDgzTHhUclBtUlhocGVsRGh1STdlWWxvYnRvNlovUUVLRVZvNmZcbkxEdDIzRzNTNndLQmdRRFJjR2Y0NkFIckJsL3JaejFvaGNGZDR0THNraHl4d2VYUEFuSTV2dkVaUmhVYTdMUEdcbkxGOHNJL25UOFZlL3VVQlRvQitLS0lSajFzeCtsbmo3YTJTNXFoS1JIWi9DbzMvTG5uRDkzU2RUWTdqZXViRitcbkFSRFVOVDRPTkRvRldkeENHWXgwSUw2TlFoSHI4dHNyclVIQjZzcnhYaWgremViaDVDMXVzeTlOelFLQmdRRFBcbmRlQUVoRnUzTnBkaTBLdjVyUkQ3amFrcm44QVFEaHFEeDMxR2tYN1RJc1VEWlY3NHQyVUFMbkJFOElkQXZNblpcbkZrVDRHMWZNZW5PRldEKzZWOEFvbkFxaEQwVHpWL3M2bE5YNzNsdWlockNyLzE4K0Jjb2VlQTM1RUUxeUNxNy9cbnRNRmdxWkcxWG85RDF3VzRCeVpVY0FGWERjMmtTVnF5WTZ4TTRGZ0J1d0tCZ1FEQ1AxclF4K2ZoNWdrY1BNUHBcbitYZS9oMDR1SmRSdmpaUkE0MFo3aHViRVFReFNEeWtkMHZSRXFQVlhNS1p1OTFkb0NwRGZHd051dlFkbHY5ZzRcbmc2NDFxRkNXd2ZRb0VZZTBTWUhJVUY5NlRwazVpeGdmUlhYVHRxNGE2c0NvMGV2NG51aFh4Vk82NFRyY0E1YUVcbnB1dWMvYzVTTlh1M0plMWgwK1Jzcm1lWkh3PT1cbi0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS1cbiIsCiAgImNsaWVudF9lbWFpbCI6ICJtYXJrLWRldkBrYWxudGVyYS1kZW1vLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAiY2xpZW50X2lkIjogIjExNTExODU2MTEwMTQ1MDM0MDQ0MCIsCiAgImF1dGhfdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi9hdXRoIiwKICAidG9rZW5fdXJpIjogImh0dHBzOi8vb2F1dGgyLmdvb2dsZWFwaXMuY29tL3Rva2VuIiwKICAiYXV0aF9wcm92aWRlcl94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL29hdXRoMi92MS9jZXJ0cyIsCiAgImNsaWVudF94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3JvYm90L3YxL21ldGFkYXRhL3g1MDkvbWFyay1kZXYlNDBrYWxudGVyYS1kZW1vLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAidW5pdmVyc2VfZG9tYWluIjogImdvb2dsZWFwaXMuY29tIgp9Cg== kind: Secret metadata: name: vertexai-creds @@ -1967,7 +2027,7 @@ items: - non-persistent://tg/request/text-completion-rag - -o - non-persistent://tg/response/text-completion-rag-response - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: text-completion-rag resources: limits: @@ -2021,7 +2081,7 @@ items: - embeddings-vectorize - -p - pulsar://pulsar:6650 - image: docker.io/trustgraph/trustgraph-flow:0.10.0 + image: docker.io/trustgraph/trustgraph-flow:0.10.1 name: vectorize resources: limits: