apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-05-30 01:35:14 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

- Some protection in extract-definition

Browse source 
- Up version
This commit is contained in:
Cyber MacGeddon 2024-07-25 22:44:51 +01:00
parent 354e95cb1c
commit 4bc215e19a
7 changed files with 54 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -1,6 +1,6 @@
# VERSION=$(shell git describe | sed 's/^v//') # VERSION=$(shell git describe | sed 's/^v//')
VERSION=0.5.2 VERSION=0.5.3
all: container all: container

20
docker-compose-azure.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100 restart: on-failure:100
pdf-decoder: pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100 restart: on-failure:100
chunker: chunker:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vectorize: vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100 restart: on-failure:100
embeddings: embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-definitions: kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-relationships: kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
@ -169,7 +169,7 @@ services:
restart: on-failure:100 restart: on-failure:100
store-graph-embeddings: store-graph-embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "ge-write-milvus" - "ge-write-milvus"
- "-p" - "-p"
@ -179,7 +179,7 @@ services:
restart: on-failure:100 restart: on-failure:100
store-triples: store-triples:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "triples-write-cassandra" - "triples-write-cassandra"
- "-p" - "-p"
@ -189,7 +189,7 @@ services:
restart: on-failure:100 restart: on-failure:100
text-completion: text-completion:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "text-completion-azure" - "text-completion-azure"
- "-p" - "-p"
@ -201,7 +201,7 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-rag: graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "graph-rag" - "graph-rag"
- "-p" - "-p"

20
docker-compose-claude.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100 restart: on-failure:100
pdf-decoder: pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100 restart: on-failure:100
chunker: chunker:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vectorize: vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100 restart: on-failure:100
embeddings: embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-definitions: kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-relationships: kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
@ -169,7 +169,7 @@ services:
restart: on-failure:100 restart: on-failure:100
store-graph-embeddings: store-graph-embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "ge-write-milvus" - "ge-write-milvus"
- "-p" - "-p"
@ -179,7 +179,7 @@ services:
restart: on-failure:100 restart: on-failure:100
store-triples: store-triples:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "triples-write-cassandra" - "triples-write-cassandra"
- "-p" - "-p"
@ -189,7 +189,7 @@ services:
restart: on-failure:100 restart: on-failure:100
text-completion: text-completion:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "text-completion-claude" - "text-completion-claude"
- "-p" - "-p"
@ -199,7 +199,7 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-rag: graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "graph-rag" - "graph-rag"
- "-p" - "-p"

22
docker-compose-ollama.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100 restart: on-failure:100
pdf-decoder: pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100 restart: on-failure:100
chunker: chunker:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vectorize: vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100 restart: on-failure:100
embeddings: embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-definitions: kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-relationships: kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
@ -169,7 +169,7 @@ services:
restart: on-failure:100 restart: on-failure:100
store-graph-embeddings: store-graph-embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "ge-write-milvus" - "ge-write-milvus"
- "-p" - "-p"
@ -179,7 +179,7 @@ services:
restart: on-failure:100 restart: on-failure:100
store-triples: store-triples:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "triples-write-cassandra" - "triples-write-cassandra"
- "-p" - "-p"
@ -189,17 +189,19 @@ services:
restart: on-failure:100 restart: on-failure:100
text-completion: text-completion:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "text-completion-ollama" - "text-completion-ollama"
- "-p" - "-p"
- "pulsar://pulsar:6650" - "pulsar://pulsar:6650"
- "-r" - "-r"
- "http://${OLLAMA_HOST}:11434/" - "http://${OLLAMA_HOST}:11434/"
# - "-m"
# - "llama3.1:8b"
restart: on-failure:100 restart: on-failure:100
graph-rag: graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "graph-rag" - "graph-rag"
- "-p" - "-p"

20
docker-compose-vertexai.yaml
View file

@ -119,7 +119,7 @@ services:
restart: on-failure:100 restart: on-failure:100
pdf-decoder: pdf-decoder:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "pdf-decoder" - "pdf-decoder"
- "-p" - "-p"
@ -127,7 +127,7 @@ services:
restart: on-failure:100 restart: on-failure:100
chunker: chunker:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "chunker-recursive" - "chunker-recursive"
- "-p" - "-p"
@ -135,7 +135,7 @@ services:
restart: on-failure:100 restart: on-failure:100
vectorize: vectorize:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "embeddings-vectorize" - "embeddings-vectorize"
- "-p" - "-p"
@ -143,7 +143,7 @@ services:
restart: on-failure:100 restart: on-failure:100
embeddings: embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "embeddings-hf" - "embeddings-hf"
- "-p" - "-p"
@ -153,7 +153,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-definitions: kg-extract-definitions:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "kg-extract-definitions" - "kg-extract-definitions"
- "-p" - "-p"
@ -161,7 +161,7 @@ services:
restart: on-failure:100 restart: on-failure:100
kg-extract-relationships: kg-extract-relationships:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "kg-extract-relationships" - "kg-extract-relationships"
- "-p" - "-p"
@ -169,7 +169,7 @@ services:
restart: on-failure:100 restart: on-failure:100
store-graph-embeddings: store-graph-embeddings:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "ge-write-milvus" - "ge-write-milvus"
- "-p" - "-p"
@ -179,7 +179,7 @@ services:
restart: on-failure:100 restart: on-failure:100
store-triples: store-triples:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "triples-write-cassandra" - "triples-write-cassandra"
- "-p" - "-p"
@ -189,7 +189,7 @@ services:
restart: on-failure:100 restart: on-failure:100
text-completion: text-completion:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "text-completion-ollama" - "text-completion-ollama"
- "-p" - "-p"
@ -203,7 +203,7 @@ services:
restart: on-failure:100 restart: on-failure:100
graph-rag: graph-rag:
image: docker.io/trustgraph/trustgraph-flow:0.5.2 image: docker.io/trustgraph/trustgraph-flow:0.5.3
command: command:
- "graph-rag" - "graph-rag"
- "-p" - "-p"

11
setup.py
View file

@ -4,7 +4,7 @@ import os
with open("README.md", "r") as fh: with open("README.md", "r") as fh:
long_description = fh.read() long_description = fh.read()
version = "0.5.2" version = "0.5.3"
setuptools.setup( setuptools.setup(
name="trustgraph", name="trustgraph",
@ -48,9 +48,12 @@ setuptools.setup(
], ],
scripts=[ scripts=[
"scripts/chunker-recursive", "scripts/chunker-recursive",
"scripts/concat-parquet",
"scripts/dump-parquet",
"scripts/embeddings-hf", "scripts/embeddings-hf",
"scripts/embeddings-ollama", "scripts/embeddings-ollama",
"scripts/embeddings-vectorize", "scripts/embeddings-vectorize",
"scripts/ge-dump-parquet",
"scripts/ge-write-milvus", "scripts/ge-write-milvus",
"scripts/graph-rag", "scripts/graph-rag",
"scripts/graph-show", "scripts/graph-show",
@ -58,6 +61,8 @@ setuptools.setup(
"scripts/init-pulsar-manager", "scripts/init-pulsar-manager",
"scripts/kg-extract-definitions", "scripts/kg-extract-definitions",
"scripts/kg-extract-relationships", "scripts/kg-extract-relationships",
"scripts/load-graph-embeddings",
"scripts/load-triples",
"scripts/loader", "scripts/loader",
"scripts/pdf-decoder", "scripts/pdf-decoder",
"scripts/query", "scripts/query",
@ -66,9 +71,7 @@ setuptools.setup(
"scripts/text-completion-claude", "scripts/text-completion-claude",
"scripts/text-completion-ollama", "scripts/text-completion-ollama",
"scripts/text-completion-vertexai", "scripts/text-completion-vertexai",
"scripts/triples-write-cassandra",
"scripts/dump-parquet",
"scripts/triples-dump-parquet", "scripts/triples-dump-parquet",
"scripts/ge-dump-parquet", "scripts/triples-write-cassandra",
] ]
) )

4
trustgraph/kg/extract_relationships/extract.py
View file

@ -111,6 +111,10 @@ class Processor(ConsumerProducer):
p = rel["predicate"] p = rel["predicate"]
o = rel["object"] o = rel["object"]
if s == "": continue
if p == "": continue
if o == "": continue
s_uri = self.to_uri(s) s_uri = self.to_uri(s)
s_value = Value(value=str(s_uri), is_uri=True) s_value = Value(value=str(s_uri), is_uri=True)