Parquet split

Containerfile

@@ -40,6 +40,9 @@ COPY README.md /root/build/trustgraph-vertexai/
 COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
 COPY README.md /root/build/trustgraph-bedrock/
 
+COPY trustgraph-parquet/ /root/build/trustgraph-parquet/
+COPY README.md /root/build/trustgraph-parquet/
+
 COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/
 COPY README.md /root/build/trustgraph-embeddings-hf/
 
@@ -49,6 +52,7 @@ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-parquet/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/
 
 RUN ls /root/wheels
@@ -66,6 +70,7 @@ RUN \
     pip3 install /root/wheels/trustgraph_flow-* && \
     pip3 install /root/wheels/trustgraph_vertexai-* && \
     pip3 install /root/wheels/trustgraph_bedrock-* && \
+    pip3 install /root/wheels/trustgraph_parquet-* && \
     pip3 install /root/wheels/trustgraph_embeddings_hf-* && \
     pip3 cache purge && \
     rm -rf /root/wheels

Makefile

@@ -11,6 +11,7 @@ pkgs:
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-flow/
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-vertexai/
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-bedrock/
+	pip3 wheel --no-deps --wheel-dir dist trustgraph-parquet/
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/
 
 CONTAINER=docker.io/trustgraph/trustgraph-flow
@@ -20,6 +21,7 @@ update-package-versions:
 	echo __version__ = \"${VERSION}\" > trustgraph-flow/trustgraph/flow_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-vertexai/trustgraph/vertexai_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-bedrock/trustgraph/bedrock_version.py
+	echo __version__ = \"${VERSION}\" > trustgraph-parquet/trustgraph/parquet_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py
 
 container: update-package-versions

trustgraph-flow/setup.py

@@ -51,7 +51,6 @@ setuptools.setup(
         "anthropic",
         "pyyaml",
         "prometheus-client",
-        "pyarrow",
         "cohere",
         "openai",
         "neo4j",
@@ -60,16 +59,13 @@ setuptools.setup(
     scripts=[
         "scripts/chunker-recursive",
         "scripts/chunker-token",
-        "scripts/concat-parquet",
         "scripts/de-query-milvus",
         "scripts/de-query-qdrant",
         "scripts/de-write-milvus",
         "scripts/de-write-qdrant",
         "scripts/document-rag",
-        "scripts/dump-parquet",
         "scripts/embeddings-ollama",
         "scripts/embeddings-vectorize",
-        "scripts/ge-dump-parquet",
         "scripts/ge-query-milvus",
         "scripts/ge-query-qdrant",
         "scripts/ge-write-milvus",
@@ -81,10 +77,8 @@ setuptools.setup(
         "scripts/kg-extract-definitions",
         "scripts/kg-extract-topics",
         "scripts/kg-extract-relationships",
-        "scripts/load-graph-embeddings",
         "scripts/load-pdf",
         "scripts/load-text",
-        "scripts/load-triples",
         "scripts/metering",
         "scripts/object-extract-row",
         "scripts/oe-write-milvus",
@@ -103,7 +97,6 @@ setuptools.setup(
         "scripts/text-completion-openai",
         "scripts/tg-init-pulsar",
         "scripts/tg-processor-state",
-        "scripts/triples-dump-parquet",
         "scripts/triples-query-cassandra",
         "scripts/triples-query-neo4j",
         "scripts/triples-write-cassandra",

trustgraph-parquet/setup.py

@@ -0,0 +1,48 @@
+import setuptools
+import os
+import importlib
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+# Load a version number module
+spec = importlib.util.spec_from_file_location(
+    'version', 'trustgraph/parquet_version.py'
+)
+version_module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(version_module)
+
+version = version_module.__version__
+
+setuptools.setup(
+    name="trustgraph-parquet",
+    version=version,
+    author="trustgraph.ai",
+    author_email="security@trustgraph.ai",
+    description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/trustgraph-ai/trustgraph",
+    packages=setuptools.find_namespace_packages(
+        where='./',
+    ),
+    classifiers=[ 
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "Operating System :: OS Independent",
+    ],
+    python_requires='>=3.8',
+    download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
+    install_requires=[
+        "trustgraph-base",
+        "pulsar-client",
+        "prometheus-client",
+        "pyarrow",
+    ],
+    scripts=[
+        "scripts/concat-parquet",
+        "scripts/dump-parquet",
+        "scripts/ge-dump-parquet",
+        "scripts/triples-dump-parquet",
+    ]
+)