From 012b6f76723ffa92a6314cf9797ed8d42d9411c7 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Mon, 30 Sep 2024 17:54:12 +0100 Subject: [PATCH] Parquet split --- Containerfile | 5 ++ Makefile | 2 + trustgraph-flow/setup.py | 7 --- .../scripts/concat-parquet | 0 .../scripts/dump-parquet | 0 .../scripts/ge-dump-parquet | 0 .../scripts/load-graph-embeddings | 0 .../scripts/load-triples | 0 .../scripts/triples-dump-parquet | 0 trustgraph-parquet/setup.py | 48 +++++++++++++++++++ .../trustgraph/dump/__init__.py | 0 .../dump/graph_embeddings/__init__.py | 0 .../dump/graph_embeddings/parquet/__init__.py | 0 .../dump/graph_embeddings/parquet/__main__.py | 0 .../graph_embeddings/parquet/processor.py | 0 .../dump/graph_embeddings/parquet/writer.py | 0 .../trustgraph/dump/triples/__init__.py | 0 .../dump/triples/parquet/__init__.py | 0 .../dump/triples/parquet/__main__.py | 0 .../dump/triples/parquet/processor.py | 0 .../trustgraph/dump/triples/parquet/writer.py | 0 21 files changed, 55 insertions(+), 7 deletions(-) rename {trustgraph-flow => trustgraph-parquet}/scripts/concat-parquet (100%) rename {trustgraph-flow => trustgraph-parquet}/scripts/dump-parquet (100%) rename {trustgraph-flow => trustgraph-parquet}/scripts/ge-dump-parquet (100%) rename {trustgraph-flow => trustgraph-parquet}/scripts/load-graph-embeddings (100%) rename {trustgraph-flow => trustgraph-parquet}/scripts/load-triples (100%) rename {trustgraph-flow => trustgraph-parquet}/scripts/triples-dump-parquet (100%) create mode 100644 trustgraph-parquet/setup.py rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/__init__.py (100%) rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/graph_embeddings/__init__.py (100%) rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/graph_embeddings/parquet/__init__.py (100%) rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/graph_embeddings/parquet/__main__.py (100%) rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/graph_embeddings/parquet/processor.py (100%) rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/graph_embeddings/parquet/writer.py (100%) rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/triples/__init__.py (100%) rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/triples/parquet/__init__.py (100%) rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/triples/parquet/__main__.py (100%) rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/triples/parquet/processor.py (100%) rename {trustgraph-flow => trustgraph-parquet}/trustgraph/dump/triples/parquet/writer.py (100%) diff --git a/Containerfile b/Containerfile index 66d15786..ef0074d6 100644 --- a/Containerfile +++ b/Containerfile @@ -40,6 +40,9 @@ COPY README.md /root/build/trustgraph-vertexai/ COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/ COPY README.md /root/build/trustgraph-bedrock/ +COPY trustgraph-parquet/ /root/build/trustgraph-parquet/ +COPY README.md /root/build/trustgraph-parquet/ + COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/ COPY README.md /root/build/trustgraph-embeddings-hf/ @@ -49,6 +52,7 @@ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-parquet/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/ RUN ls /root/wheels @@ -66,6 +70,7 @@ RUN \ pip3 install /root/wheels/trustgraph_flow-* && \ pip3 install /root/wheels/trustgraph_vertexai-* && \ pip3 install /root/wheels/trustgraph_bedrock-* && \ + pip3 install /root/wheels/trustgraph_parquet-* && \ pip3 install /root/wheels/trustgraph_embeddings_hf-* && \ pip3 cache purge && \ rm -rf /root/wheels diff --git a/Makefile b/Makefile index 55bd13d9..25b48dd2 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ pkgs: pip3 wheel --no-deps --wheel-dir dist trustgraph-flow/ pip3 wheel --no-deps --wheel-dir dist trustgraph-vertexai/ pip3 wheel --no-deps --wheel-dir dist trustgraph-bedrock/ + pip3 wheel --no-deps --wheel-dir dist trustgraph-parquet/ pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/ CONTAINER=docker.io/trustgraph/trustgraph-flow @@ -20,6 +21,7 @@ update-package-versions: echo __version__ = \"${VERSION}\" > trustgraph-flow/trustgraph/flow_version.py echo __version__ = \"${VERSION}\" > trustgraph-vertexai/trustgraph/vertexai_version.py echo __version__ = \"${VERSION}\" > trustgraph-bedrock/trustgraph/bedrock_version.py + echo __version__ = \"${VERSION}\" > trustgraph-parquet/trustgraph/parquet_version.py echo __version__ = \"${VERSION}\" > trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py container: update-package-versions diff --git a/trustgraph-flow/setup.py b/trustgraph-flow/setup.py index 3b66d01d..ce395c97 100644 --- a/trustgraph-flow/setup.py +++ b/trustgraph-flow/setup.py @@ -51,7 +51,6 @@ setuptools.setup( "anthropic", "pyyaml", "prometheus-client", - "pyarrow", "cohere", "openai", "neo4j", @@ -60,16 +59,13 @@ setuptools.setup( scripts=[ "scripts/chunker-recursive", "scripts/chunker-token", - "scripts/concat-parquet", "scripts/de-query-milvus", "scripts/de-query-qdrant", "scripts/de-write-milvus", "scripts/de-write-qdrant", "scripts/document-rag", - "scripts/dump-parquet", "scripts/embeddings-ollama", "scripts/embeddings-vectorize", - "scripts/ge-dump-parquet", "scripts/ge-query-milvus", "scripts/ge-query-qdrant", "scripts/ge-write-milvus", @@ -81,10 +77,8 @@ setuptools.setup( "scripts/kg-extract-definitions", "scripts/kg-extract-topics", "scripts/kg-extract-relationships", - "scripts/load-graph-embeddings", "scripts/load-pdf", "scripts/load-text", - "scripts/load-triples", "scripts/metering", "scripts/object-extract-row", "scripts/oe-write-milvus", @@ -103,7 +97,6 @@ setuptools.setup( "scripts/text-completion-openai", "scripts/tg-init-pulsar", "scripts/tg-processor-state", - "scripts/triples-dump-parquet", "scripts/triples-query-cassandra", "scripts/triples-query-neo4j", "scripts/triples-write-cassandra", diff --git a/trustgraph-flow/scripts/concat-parquet b/trustgraph-parquet/scripts/concat-parquet similarity index 100% rename from trustgraph-flow/scripts/concat-parquet rename to trustgraph-parquet/scripts/concat-parquet diff --git a/trustgraph-flow/scripts/dump-parquet b/trustgraph-parquet/scripts/dump-parquet similarity index 100% rename from trustgraph-flow/scripts/dump-parquet rename to trustgraph-parquet/scripts/dump-parquet diff --git a/trustgraph-flow/scripts/ge-dump-parquet b/trustgraph-parquet/scripts/ge-dump-parquet similarity index 100% rename from trustgraph-flow/scripts/ge-dump-parquet rename to trustgraph-parquet/scripts/ge-dump-parquet diff --git a/trustgraph-flow/scripts/load-graph-embeddings b/trustgraph-parquet/scripts/load-graph-embeddings similarity index 100% rename from trustgraph-flow/scripts/load-graph-embeddings rename to trustgraph-parquet/scripts/load-graph-embeddings diff --git a/trustgraph-flow/scripts/load-triples b/trustgraph-parquet/scripts/load-triples similarity index 100% rename from trustgraph-flow/scripts/load-triples rename to trustgraph-parquet/scripts/load-triples diff --git a/trustgraph-flow/scripts/triples-dump-parquet b/trustgraph-parquet/scripts/triples-dump-parquet similarity index 100% rename from trustgraph-flow/scripts/triples-dump-parquet rename to trustgraph-parquet/scripts/triples-dump-parquet diff --git a/trustgraph-parquet/setup.py b/trustgraph-parquet/setup.py new file mode 100644 index 00000000..6da7d916 --- /dev/null +++ b/trustgraph-parquet/setup.py @@ -0,0 +1,48 @@ +import setuptools +import os +import importlib + +with open("README.md", "r") as fh: + long_description = fh.read() + +# Load a version number module +spec = importlib.util.spec_from_file_location( + 'version', 'trustgraph/parquet_version.py' +) +version_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(version_module) + +version = version_module.__version__ + +setuptools.setup( + name="trustgraph-parquet", + version=version, + author="trustgraph.ai", + author_email="security@trustgraph.ai", + description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/trustgraph-ai/trustgraph", + packages=setuptools.find_namespace_packages( + where='./', + ), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Operating System :: OS Independent", + ], + python_requires='>=3.8', + download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", + install_requires=[ + "trustgraph-base", + "pulsar-client", + "prometheus-client", + "pyarrow", + ], + scripts=[ + "scripts/concat-parquet", + "scripts/dump-parquet", + "scripts/ge-dump-parquet", + "scripts/triples-dump-parquet", + ] +) diff --git a/trustgraph-flow/trustgraph/dump/__init__.py b/trustgraph-parquet/trustgraph/dump/__init__.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/__init__.py rename to trustgraph-parquet/trustgraph/dump/__init__.py diff --git a/trustgraph-flow/trustgraph/dump/graph_embeddings/__init__.py b/trustgraph-parquet/trustgraph/dump/graph_embeddings/__init__.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/graph_embeddings/__init__.py rename to trustgraph-parquet/trustgraph/dump/graph_embeddings/__init__.py diff --git a/trustgraph-flow/trustgraph/dump/graph_embeddings/parquet/__init__.py b/trustgraph-parquet/trustgraph/dump/graph_embeddings/parquet/__init__.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/graph_embeddings/parquet/__init__.py rename to trustgraph-parquet/trustgraph/dump/graph_embeddings/parquet/__init__.py diff --git a/trustgraph-flow/trustgraph/dump/graph_embeddings/parquet/__main__.py b/trustgraph-parquet/trustgraph/dump/graph_embeddings/parquet/__main__.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/graph_embeddings/parquet/__main__.py rename to trustgraph-parquet/trustgraph/dump/graph_embeddings/parquet/__main__.py diff --git a/trustgraph-flow/trustgraph/dump/graph_embeddings/parquet/processor.py b/trustgraph-parquet/trustgraph/dump/graph_embeddings/parquet/processor.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/graph_embeddings/parquet/processor.py rename to trustgraph-parquet/trustgraph/dump/graph_embeddings/parquet/processor.py diff --git a/trustgraph-flow/trustgraph/dump/graph_embeddings/parquet/writer.py b/trustgraph-parquet/trustgraph/dump/graph_embeddings/parquet/writer.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/graph_embeddings/parquet/writer.py rename to trustgraph-parquet/trustgraph/dump/graph_embeddings/parquet/writer.py diff --git a/trustgraph-flow/trustgraph/dump/triples/__init__.py b/trustgraph-parquet/trustgraph/dump/triples/__init__.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/triples/__init__.py rename to trustgraph-parquet/trustgraph/dump/triples/__init__.py diff --git a/trustgraph-flow/trustgraph/dump/triples/parquet/__init__.py b/trustgraph-parquet/trustgraph/dump/triples/parquet/__init__.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/triples/parquet/__init__.py rename to trustgraph-parquet/trustgraph/dump/triples/parquet/__init__.py diff --git a/trustgraph-flow/trustgraph/dump/triples/parquet/__main__.py b/trustgraph-parquet/trustgraph/dump/triples/parquet/__main__.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/triples/parquet/__main__.py rename to trustgraph-parquet/trustgraph/dump/triples/parquet/__main__.py diff --git a/trustgraph-flow/trustgraph/dump/triples/parquet/processor.py b/trustgraph-parquet/trustgraph/dump/triples/parquet/processor.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/triples/parquet/processor.py rename to trustgraph-parquet/trustgraph/dump/triples/parquet/processor.py diff --git a/trustgraph-flow/trustgraph/dump/triples/parquet/writer.py b/trustgraph-parquet/trustgraph/dump/triples/parquet/writer.py similarity index 100% rename from trustgraph-flow/trustgraph/dump/triples/parquet/writer.py rename to trustgraph-parquet/trustgraph/dump/triples/parquet/writer.py