Utils

Containerfile

@@ -46,6 +46,9 @@ COPY README.md /root/build/trustgraph-parquet/
 COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/
 COPY README.md /root/build/trustgraph-embeddings-hf/
 
+COPY trustgraph-utils/ /root/build/trustgraph-utils/
+COPY README.md /root/build/trustgraph-utils/
+
 WORKDIR /root/build/
 
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
@@ -54,6 +57,7 @@ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-parquet/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-utils/
 
 RUN ls /root/wheels
 
@@ -72,6 +76,7 @@ RUN \
     pip3 install /root/wheels/trustgraph_bedrock-* && \
     pip3 install /root/wheels/trustgraph_parquet-* && \
     pip3 install /root/wheels/trustgraph_embeddings_hf-* && \
+    pip3 install /root/wheels/trustgraph_utils-* && \
     pip3 cache purge && \
     rm -rf /root/wheels
 

Makefile

@@ -13,16 +13,19 @@ pkgs:
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-bedrock/
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-parquet/
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/
+	pip3 wheel --no-deps --wheel-dir dist trustgraph-utils/
 
 CONTAINER=docker.io/trustgraph/trustgraph-flow
 
 update-package-versions:
+	mkdir -p trustgraph-utils/trustgraph
 	echo __version__ = \"${VERSION}\" > trustgraph-base/trustgraph/base_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-flow/trustgraph/flow_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-vertexai/trustgraph/vertexai_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-bedrock/trustgraph/bedrock_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-parquet/trustgraph/parquet_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py
+	echo __version__ = \"${VERSION}\" > trustgraph-utils/trustgraph/utils_version.py
 
 container: update-package-versions
 	${DOCKER} build -f Containerfile -t ${CONTAINER}:${VERSION} \

trustgraph-flow/setup.py

@@ -71,22 +71,15 @@ setuptools.setup(
         "scripts/ge-write-milvus",
         "scripts/ge-write-qdrant",
         "scripts/graph-rag",
-        "scripts/graph-show",
-        "scripts/graph-to-turtle",
-        "scripts/init-pulsar-manager",
         "scripts/kg-extract-definitions",
         "scripts/kg-extract-topics",
         "scripts/kg-extract-relationships",
-        "scripts/load-pdf",
-        "scripts/load-text",
         "scripts/metering",
         "scripts/object-extract-row",
         "scripts/oe-write-milvus",
         "scripts/pdf-decoder",
         "scripts/prompt-generic",
         "scripts/prompt-template",
-        "scripts/query-document-rag",
-        "scripts/query-graph-rag",
         "scripts/rows-write-cassandra",
         "scripts/run-processing",
         "scripts/text-completion-azure",
@@ -95,8 +88,6 @@ setuptools.setup(
         "scripts/text-completion-llamafile",
         "scripts/text-completion-ollama",
         "scripts/text-completion-openai",
-        "scripts/tg-init-pulsar",
-        "scripts/tg-processor-state",
         "scripts/triples-query-cassandra",
         "scripts/triples-query-neo4j",
         "scripts/triples-write-cassandra",

trustgraph-utils/setup.py

@@ -1,13 +1,21 @@
 import setuptools
 import os
+import importlib
 
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
-version = "0.11.7"
+# Load a version number module
+spec = importlib.util.spec_from_file_location(
+    'version', 'trustgraph/utils_version.py'
+)
+version_module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(version_module)
+
+version = version_module.__version__
 
 setuptools.setup(
-    name="trustgraph-core",
+    name="trustgraph-utils",
     version=version,
     author="trustgraph.ai",
     author_email="security@trustgraph.ai",
@@ -17,7 +25,6 @@ setuptools.setup(
     url="https://github.com/trustgraph-ai/trustgraph",
     packages=setuptools.find_namespace_packages(
         where='./',
-#         include=['trustgraph.core']
     ),
     classifiers=[ 
         "Programming Language :: Python :: 3",
@@ -27,82 +34,20 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "urllib3",
-        "rdflib",
-        "pymilvus",
-        "langchain",
-        "langchain-core",
-        "langchain-text-splitters",
-        "langchain-community",
+        "trustgraph-base",
         "requests",
-        "cassandra-driver",
         "pulsar-client",
-        "pypdf",
-        "qdrant-client",
         "tabulate",
-        "anthropic",
-        "google-cloud-aiplatform",
-        "pyyaml",
-        "prometheus-client",
-        "pyarrow",
-        "cohere",
-        "boto3",
-        "openai",
-        "neo4j",
-        "tiktoken",
     ],
     scripts=[
-        "scripts/chunker-recursive",
-        "scripts/chunker-token",
-        "scripts/concat-parquet",
-        "scripts/de-query-milvus",
-        "scripts/de-query-qdrant",
-        "scripts/de-write-milvus",
-        "scripts/de-write-qdrant",
-        "scripts/document-rag",
-        "scripts/dump-parquet",
-        "scripts/embeddings-ollama",
-        "scripts/embeddings-vectorize",
-        "scripts/ge-dump-parquet",
-        "scripts/ge-query-milvus",
-        "scripts/ge-query-qdrant",
-        "scripts/ge-write-milvus",
-        "scripts/ge-write-qdrant",
-        "scripts/graph-rag",
         "scripts/graph-show",
         "scripts/graph-to-turtle",
         "scripts/init-pulsar-manager",
-        "scripts/kg-extract-definitions",
-        "scripts/kg-extract-topics",
-        "scripts/kg-extract-relationships",
-        "scripts/load-graph-embeddings",
         "scripts/load-pdf",
         "scripts/load-text",
-        "scripts/load-triples",
-        "scripts/metering",
-        "scripts/object-extract-row",
-        "scripts/oe-write-milvus",
-        "scripts/pdf-decoder",
-        "scripts/prompt-generic",
-        "scripts/prompt-template",
         "scripts/query-document-rag",
         "scripts/query-graph-rag",
-        "scripts/rows-write-cassandra",
-        "scripts/run-processing",
-        "scripts/text-completion-azure",
-        "scripts/text-completion-bedrock",
-        "scripts/text-completion-claude",
-        "scripts/text-completion-cohere",
-        "scripts/text-completion-llamafile",
-        "scripts/text-completion-ollama",
-        "scripts/text-completion-openai",
-        "scripts/text-completion-vertexai",
         "scripts/tg-init-pulsar",
         "scripts/tg-processor-state",
-        "scripts/triples-dump-parquet",
-        "scripts/triples-query-cassandra",
-        "scripts/triples-query-neo4j",
-        "scripts/triples-write-cassandra",
-        "scripts/triples-write-neo4j",
     ]
 )