diff --git a/Makefile b/Makefile index 5f19f05e..ef182fe4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # VERSION=$(shell git describe | sed 's/^v//') -VERSION=0.5.2 +VERSION=0.5.3 all: container diff --git a/docker-compose-azure.yaml b/docker-compose-azure.yaml index ac397436..7b474841 100644 --- a/docker-compose-azure.yaml +++ b/docker-compose-azure.yaml @@ -119,7 +119,7 @@ services: restart: on-failure:100 pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "pdf-decoder" - "-p" @@ -127,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "chunker-recursive" - "-p" @@ -135,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "embeddings-vectorize" - "-p" @@ -143,7 +143,7 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "embeddings-hf" - "-p" @@ -153,7 +153,7 @@ services: restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "kg-extract-definitions" - "-p" @@ -161,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "kg-extract-relationships" - "-p" @@ -169,7 +169,7 @@ services: restart: on-failure:100 store-graph-embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "ge-write-milvus" - "-p" @@ -179,7 +179,7 @@ services: restart: on-failure:100 store-triples: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "triples-write-cassandra" - "-p" @@ -189,7 +189,7 @@ services: restart: on-failure:100 text-completion: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "text-completion-azure" - "-p" @@ -201,7 +201,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "graph-rag" - "-p" diff --git a/docker-compose-claude.yaml b/docker-compose-claude.yaml index e980477d..26f24f3b 100644 --- a/docker-compose-claude.yaml +++ b/docker-compose-claude.yaml @@ -119,7 +119,7 @@ services: restart: on-failure:100 pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "pdf-decoder" - "-p" @@ -127,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "chunker-recursive" - "-p" @@ -135,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "embeddings-vectorize" - "-p" @@ -143,7 +143,7 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "embeddings-hf" - "-p" @@ -153,7 +153,7 @@ services: restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "kg-extract-definitions" - "-p" @@ -161,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "kg-extract-relationships" - "-p" @@ -169,7 +169,7 @@ services: restart: on-failure:100 store-graph-embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "ge-write-milvus" - "-p" @@ -179,7 +179,7 @@ services: restart: on-failure:100 store-triples: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "triples-write-cassandra" - "-p" @@ -189,7 +189,7 @@ services: restart: on-failure:100 text-completion: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "text-completion-claude" - "-p" @@ -199,7 +199,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "graph-rag" - "-p" diff --git a/docker-compose-ollama.yaml b/docker-compose-ollama.yaml index 562a6260..0b5b2bb8 100644 --- a/docker-compose-ollama.yaml +++ b/docker-compose-ollama.yaml @@ -119,7 +119,7 @@ services: restart: on-failure:100 pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "pdf-decoder" - "-p" @@ -127,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "chunker-recursive" - "-p" @@ -135,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "embeddings-vectorize" - "-p" @@ -143,7 +143,7 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "embeddings-hf" - "-p" @@ -153,7 +153,7 @@ services: restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "kg-extract-definitions" - "-p" @@ -161,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "kg-extract-relationships" - "-p" @@ -169,7 +169,7 @@ services: restart: on-failure:100 store-graph-embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "ge-write-milvus" - "-p" @@ -179,7 +179,7 @@ services: restart: on-failure:100 store-triples: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "triples-write-cassandra" - "-p" @@ -189,17 +189,19 @@ services: restart: on-failure:100 text-completion: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "text-completion-ollama" - "-p" - "pulsar://pulsar:6650" - "-r" - "http://${OLLAMA_HOST}:11434/" +# - "-m" +# - "llama3.1:8b" restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "graph-rag" - "-p" diff --git a/docker-compose-vertexai.yaml b/docker-compose-vertexai.yaml index 4bf40795..b3767b7a 100644 --- a/docker-compose-vertexai.yaml +++ b/docker-compose-vertexai.yaml @@ -119,7 +119,7 @@ services: restart: on-failure:100 pdf-decoder: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "pdf-decoder" - "-p" @@ -127,7 +127,7 @@ services: restart: on-failure:100 chunker: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "chunker-recursive" - "-p" @@ -135,7 +135,7 @@ services: restart: on-failure:100 vectorize: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "embeddings-vectorize" - "-p" @@ -143,7 +143,7 @@ services: restart: on-failure:100 embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "embeddings-hf" - "-p" @@ -153,7 +153,7 @@ services: restart: on-failure:100 kg-extract-definitions: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "kg-extract-definitions" - "-p" @@ -161,7 +161,7 @@ services: restart: on-failure:100 kg-extract-relationships: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "kg-extract-relationships" - "-p" @@ -169,7 +169,7 @@ services: restart: on-failure:100 store-graph-embeddings: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "ge-write-milvus" - "-p" @@ -179,7 +179,7 @@ services: restart: on-failure:100 store-triples: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "triples-write-cassandra" - "-p" @@ -189,7 +189,7 @@ services: restart: on-failure:100 text-completion: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "text-completion-ollama" - "-p" @@ -203,7 +203,7 @@ services: restart: on-failure:100 graph-rag: - image: docker.io/trustgraph/trustgraph-flow:0.5.2 + image: docker.io/trustgraph/trustgraph-flow:0.5.3 command: - "graph-rag" - "-p" diff --git a/setup.py b/setup.py index 6d57c9a0..b25b165d 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ import os with open("README.md", "r") as fh: long_description = fh.read() -version = "0.5.2" +version = "0.5.3" setuptools.setup( name="trustgraph", @@ -48,9 +48,12 @@ setuptools.setup( ], scripts=[ "scripts/chunker-recursive", + "scripts/concat-parquet", + "scripts/dump-parquet", "scripts/embeddings-hf", "scripts/embeddings-ollama", "scripts/embeddings-vectorize", + "scripts/ge-dump-parquet", "scripts/ge-write-milvus", "scripts/graph-rag", "scripts/graph-show", @@ -58,6 +61,8 @@ setuptools.setup( "scripts/init-pulsar-manager", "scripts/kg-extract-definitions", "scripts/kg-extract-relationships", + "scripts/load-graph-embeddings", + "scripts/load-triples", "scripts/loader", "scripts/pdf-decoder", "scripts/query", @@ -66,9 +71,7 @@ setuptools.setup( "scripts/text-completion-claude", "scripts/text-completion-ollama", "scripts/text-completion-vertexai", - "scripts/triples-write-cassandra", - "scripts/dump-parquet", "scripts/triples-dump-parquet", - "scripts/ge-dump-parquet", + "scripts/triples-write-cassandra", ] ) diff --git a/trustgraph/kg/extract_relationships/extract.py b/trustgraph/kg/extract_relationships/extract.py index 55f72491..b3e34549 100755 --- a/trustgraph/kg/extract_relationships/extract.py +++ b/trustgraph/kg/extract_relationships/extract.py @@ -111,6 +111,10 @@ class Processor(ConsumerProducer): p = rel["predicate"] o = rel["object"] + if s == "": continue + if p == "": continue + if o == "": continue + s_uri = self.to_uri(s) s_value = Value(value=str(s_uri), is_uri=True)