From 0d26ebec4fb6caf7cb224245f244d396e533f014 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Mon, 30 Sep 2024 18:05:20 +0100 Subject: [PATCH] Utils --- Containerfile | 5 ++ Makefile | 3 + trustgraph-flow/setup.py | 9 --- .../scripts/graph-show | 0 .../scripts/graph-to-turtle | 0 .../scripts/init-pulsar-manager | 0 .../scripts/load-pdf | 0 .../scripts/load-text | 0 .../scripts/query-document-rag | 0 .../scripts/query-graph-rag | 0 .../scripts/tg-init-pulsar | 0 .../scripts/tg-processor-state | 0 trustgraph-utils/setup.py | 77 +++---------------- 13 files changed, 19 insertions(+), 75 deletions(-) rename {trustgraph-flow => trustgraph-utils}/scripts/graph-show (100%) rename {trustgraph-flow => trustgraph-utils}/scripts/graph-to-turtle (100%) rename {trustgraph-flow => trustgraph-utils}/scripts/init-pulsar-manager (100%) rename {trustgraph-flow => trustgraph-utils}/scripts/load-pdf (100%) rename {trustgraph-flow => trustgraph-utils}/scripts/load-text (100%) rename {trustgraph-flow => trustgraph-utils}/scripts/query-document-rag (100%) rename {trustgraph-flow => trustgraph-utils}/scripts/query-graph-rag (100%) rename {trustgraph-flow => trustgraph-utils}/scripts/tg-init-pulsar (100%) rename {trustgraph-flow => trustgraph-utils}/scripts/tg-processor-state (100%) diff --git a/Containerfile b/Containerfile index ef0074d6..cc808838 100644 --- a/Containerfile +++ b/Containerfile @@ -46,6 +46,9 @@ COPY README.md /root/build/trustgraph-parquet/ COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/ COPY README.md /root/build/trustgraph-embeddings-hf/ +COPY trustgraph-utils/ /root/build/trustgraph-utils/ +COPY README.md /root/build/trustgraph-utils/ + WORKDIR /root/build/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/ @@ -54,6 +57,7 @@ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-parquet/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-utils/ RUN ls /root/wheels @@ -72,6 +76,7 @@ RUN \ pip3 install /root/wheels/trustgraph_bedrock-* && \ pip3 install /root/wheels/trustgraph_parquet-* && \ pip3 install /root/wheels/trustgraph_embeddings_hf-* && \ + pip3 install /root/wheels/trustgraph_utils-* && \ pip3 cache purge && \ rm -rf /root/wheels diff --git a/Makefile b/Makefile index 25b48dd2..78c1cb45 100644 --- a/Makefile +++ b/Makefile @@ -13,16 +13,19 @@ pkgs: pip3 wheel --no-deps --wheel-dir dist trustgraph-bedrock/ pip3 wheel --no-deps --wheel-dir dist trustgraph-parquet/ pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/ + pip3 wheel --no-deps --wheel-dir dist trustgraph-utils/ CONTAINER=docker.io/trustgraph/trustgraph-flow update-package-versions: + mkdir -p trustgraph-utils/trustgraph echo __version__ = \"${VERSION}\" > trustgraph-base/trustgraph/base_version.py echo __version__ = \"${VERSION}\" > trustgraph-flow/trustgraph/flow_version.py echo __version__ = \"${VERSION}\" > trustgraph-vertexai/trustgraph/vertexai_version.py echo __version__ = \"${VERSION}\" > trustgraph-bedrock/trustgraph/bedrock_version.py echo __version__ = \"${VERSION}\" > trustgraph-parquet/trustgraph/parquet_version.py echo __version__ = \"${VERSION}\" > trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py + echo __version__ = \"${VERSION}\" > trustgraph-utils/trustgraph/utils_version.py container: update-package-versions ${DOCKER} build -f Containerfile -t ${CONTAINER}:${VERSION} \ diff --git a/trustgraph-flow/setup.py b/trustgraph-flow/setup.py index ce395c97..a1c89797 100644 --- a/trustgraph-flow/setup.py +++ b/trustgraph-flow/setup.py @@ -71,22 +71,15 @@ setuptools.setup( "scripts/ge-write-milvus", "scripts/ge-write-qdrant", "scripts/graph-rag", - "scripts/graph-show", - "scripts/graph-to-turtle", - "scripts/init-pulsar-manager", "scripts/kg-extract-definitions", "scripts/kg-extract-topics", "scripts/kg-extract-relationships", - "scripts/load-pdf", - "scripts/load-text", "scripts/metering", "scripts/object-extract-row", "scripts/oe-write-milvus", "scripts/pdf-decoder", "scripts/prompt-generic", "scripts/prompt-template", - "scripts/query-document-rag", - "scripts/query-graph-rag", "scripts/rows-write-cassandra", "scripts/run-processing", "scripts/text-completion-azure", @@ -95,8 +88,6 @@ setuptools.setup( "scripts/text-completion-llamafile", "scripts/text-completion-ollama", "scripts/text-completion-openai", - "scripts/tg-init-pulsar", - "scripts/tg-processor-state", "scripts/triples-query-cassandra", "scripts/triples-query-neo4j", "scripts/triples-write-cassandra", diff --git a/trustgraph-flow/scripts/graph-show b/trustgraph-utils/scripts/graph-show similarity index 100% rename from trustgraph-flow/scripts/graph-show rename to trustgraph-utils/scripts/graph-show diff --git a/trustgraph-flow/scripts/graph-to-turtle b/trustgraph-utils/scripts/graph-to-turtle similarity index 100% rename from trustgraph-flow/scripts/graph-to-turtle rename to trustgraph-utils/scripts/graph-to-turtle diff --git a/trustgraph-flow/scripts/init-pulsar-manager b/trustgraph-utils/scripts/init-pulsar-manager similarity index 100% rename from trustgraph-flow/scripts/init-pulsar-manager rename to trustgraph-utils/scripts/init-pulsar-manager diff --git a/trustgraph-flow/scripts/load-pdf b/trustgraph-utils/scripts/load-pdf similarity index 100% rename from trustgraph-flow/scripts/load-pdf rename to trustgraph-utils/scripts/load-pdf diff --git a/trustgraph-flow/scripts/load-text b/trustgraph-utils/scripts/load-text similarity index 100% rename from trustgraph-flow/scripts/load-text rename to trustgraph-utils/scripts/load-text diff --git a/trustgraph-flow/scripts/query-document-rag b/trustgraph-utils/scripts/query-document-rag similarity index 100% rename from trustgraph-flow/scripts/query-document-rag rename to trustgraph-utils/scripts/query-document-rag diff --git a/trustgraph-flow/scripts/query-graph-rag b/trustgraph-utils/scripts/query-graph-rag similarity index 100% rename from trustgraph-flow/scripts/query-graph-rag rename to trustgraph-utils/scripts/query-graph-rag diff --git a/trustgraph-flow/scripts/tg-init-pulsar b/trustgraph-utils/scripts/tg-init-pulsar similarity index 100% rename from trustgraph-flow/scripts/tg-init-pulsar rename to trustgraph-utils/scripts/tg-init-pulsar diff --git a/trustgraph-flow/scripts/tg-processor-state b/trustgraph-utils/scripts/tg-processor-state similarity index 100% rename from trustgraph-flow/scripts/tg-processor-state rename to trustgraph-utils/scripts/tg-processor-state diff --git a/trustgraph-utils/setup.py b/trustgraph-utils/setup.py index 3965b4f0..f3337f1e 100644 --- a/trustgraph-utils/setup.py +++ b/trustgraph-utils/setup.py @@ -1,13 +1,21 @@ import setuptools import os +import importlib with open("README.md", "r") as fh: long_description = fh.read() -version = "0.11.7" +# Load a version number module +spec = importlib.util.spec_from_file_location( + 'version', 'trustgraph/utils_version.py' +) +version_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(version_module) + +version = version_module.__version__ setuptools.setup( - name="trustgraph-core", + name="trustgraph-utils", version=version, author="trustgraph.ai", author_email="security@trustgraph.ai", @@ -17,7 +25,6 @@ setuptools.setup( url="https://github.com/trustgraph-ai/trustgraph", packages=setuptools.find_namespace_packages( where='./', -# include=['trustgraph.core'] ), classifiers=[ "Programming Language :: Python :: 3", @@ -27,82 +34,20 @@ setuptools.setup( python_requires='>=3.8', download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", install_requires=[ - "urllib3", - "rdflib", - "pymilvus", - "langchain", - "langchain-core", - "langchain-text-splitters", - "langchain-community", + "trustgraph-base", "requests", - "cassandra-driver", "pulsar-client", - "pypdf", - "qdrant-client", "tabulate", - "anthropic", - "google-cloud-aiplatform", - "pyyaml", - "prometheus-client", - "pyarrow", - "cohere", - "boto3", - "openai", - "neo4j", - "tiktoken", ], scripts=[ - "scripts/chunker-recursive", - "scripts/chunker-token", - "scripts/concat-parquet", - "scripts/de-query-milvus", - "scripts/de-query-qdrant", - "scripts/de-write-milvus", - "scripts/de-write-qdrant", - "scripts/document-rag", - "scripts/dump-parquet", - "scripts/embeddings-ollama", - "scripts/embeddings-vectorize", - "scripts/ge-dump-parquet", - "scripts/ge-query-milvus", - "scripts/ge-query-qdrant", - "scripts/ge-write-milvus", - "scripts/ge-write-qdrant", - "scripts/graph-rag", "scripts/graph-show", "scripts/graph-to-turtle", "scripts/init-pulsar-manager", - "scripts/kg-extract-definitions", - "scripts/kg-extract-topics", - "scripts/kg-extract-relationships", - "scripts/load-graph-embeddings", "scripts/load-pdf", "scripts/load-text", - "scripts/load-triples", - "scripts/metering", - "scripts/object-extract-row", - "scripts/oe-write-milvus", - "scripts/pdf-decoder", - "scripts/prompt-generic", - "scripts/prompt-template", "scripts/query-document-rag", "scripts/query-graph-rag", - "scripts/rows-write-cassandra", - "scripts/run-processing", - "scripts/text-completion-azure", - "scripts/text-completion-bedrock", - "scripts/text-completion-claude", - "scripts/text-completion-cohere", - "scripts/text-completion-llamafile", - "scripts/text-completion-ollama", - "scripts/text-completion-openai", - "scripts/text-completion-vertexai", "scripts/tg-init-pulsar", "scripts/tg-processor-state", - "scripts/triples-dump-parquet", - "scripts/triples-query-cassandra", - "scripts/triples-query-neo4j", - "scripts/triples-write-cassandra", - "scripts/triples-write-neo4j", ] )