From e0866d9f920a804211a2d54d4db68c80952626d1 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Mon, 30 Sep 2024 17:42:59 +0100 Subject: [PATCH] Bedrock and VertexAI --- Containerfile | 10 ++ Makefile | 4 + .../scripts/text-completion-bedrock | 0 trustgraph-bedrock/setup.py | 45 ++++++++ .../model/text_completion/bedrock/__init__.py | 0 .../model/text_completion/bedrock/__main__.py | 0 .../model/text_completion/bedrock/llm.py | 0 trustgraph-flow/setup.py | 4 - trustgraph-utils/setup.py | 108 ++++++++++++++++++ .../scripts/text-completion-vertexai | 0 trustgraph-vertexai/setup.py | 45 ++++++++ .../text-completion}/vertexai/__init__.py | 0 .../text-completion}/vertexai/__main__.py | 0 .../model/text-completion}/vertexai/llm.py | 0 14 files changed, 212 insertions(+), 4 deletions(-) rename {trustgraph-flow => trustgraph-bedrock}/scripts/text-completion-bedrock (100%) create mode 100644 trustgraph-bedrock/setup.py rename {trustgraph-flow => trustgraph-bedrock}/trustgraph/model/text_completion/bedrock/__init__.py (100%) rename {trustgraph-flow => trustgraph-bedrock}/trustgraph/model/text_completion/bedrock/__main__.py (100%) rename {trustgraph-flow => trustgraph-bedrock}/trustgraph/model/text_completion/bedrock/llm.py (100%) create mode 100644 trustgraph-utils/setup.py rename {trustgraph-flow => trustgraph-vertexai}/scripts/text-completion-vertexai (100%) create mode 100644 trustgraph-vertexai/setup.py rename {trustgraph-flow/trustgraph/model/text_completion => trustgraph-vertexai/trustgraph/model/text-completion}/vertexai/__init__.py (100%) rename {trustgraph-flow/trustgraph/model/text_completion => trustgraph-vertexai/trustgraph/model/text-completion}/vertexai/__main__.py (100%) rename {trustgraph-flow/trustgraph/model/text_completion => trustgraph-vertexai/trustgraph/model/text-completion}/vertexai/llm.py (100%) diff --git a/Containerfile b/Containerfile index 9a1ee7cd..66d15786 100644 --- a/Containerfile +++ b/Containerfile @@ -34,6 +34,12 @@ COPY README.md /root/build/trustgraph-base/ COPY trustgraph-flow/ /root/build/trustgraph-flow/ COPY README.md /root/build/trustgraph-flow/ +COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/ +COPY README.md /root/build/trustgraph-vertexai/ + +COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/ +COPY README.md /root/build/trustgraph-bedrock/ + COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/ COPY README.md /root/build/trustgraph-embeddings-hf/ @@ -41,6 +47,8 @@ WORKDIR /root/build/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/ RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/ RUN ls /root/wheels @@ -56,6 +64,8 @@ COPY --from=build /root/wheels /root/wheels RUN \ pip3 install /root/wheels/trustgraph_base-* && \ pip3 install /root/wheels/trustgraph_flow-* && \ + pip3 install /root/wheels/trustgraph_vertexai-* && \ + pip3 install /root/wheels/trustgraph_bedrock-* && \ pip3 install /root/wheels/trustgraph_embeddings_hf-* && \ pip3 cache purge && \ rm -rf /root/wheels diff --git a/Makefile b/Makefile index f4a425ec..55bd13d9 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ all: container pkgs: pip3 wheel --no-deps --wheel-dir dist trustgraph-base/ pip3 wheel --no-deps --wheel-dir dist trustgraph-flow/ + pip3 wheel --no-deps --wheel-dir dist trustgraph-vertexai/ + pip3 wheel --no-deps --wheel-dir dist trustgraph-bedrock/ pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/ CONTAINER=docker.io/trustgraph/trustgraph-flow @@ -16,6 +18,8 @@ CONTAINER=docker.io/trustgraph/trustgraph-flow update-package-versions: echo __version__ = \"${VERSION}\" > trustgraph-base/trustgraph/base_version.py echo __version__ = \"${VERSION}\" > trustgraph-flow/trustgraph/flow_version.py + echo __version__ = \"${VERSION}\" > trustgraph-vertexai/trustgraph/vertexai_version.py + echo __version__ = \"${VERSION}\" > trustgraph-bedrock/trustgraph/bedrock_version.py echo __version__ = \"${VERSION}\" > trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py container: update-package-versions diff --git a/trustgraph-flow/scripts/text-completion-bedrock b/trustgraph-bedrock/scripts/text-completion-bedrock similarity index 100% rename from trustgraph-flow/scripts/text-completion-bedrock rename to trustgraph-bedrock/scripts/text-completion-bedrock diff --git a/trustgraph-bedrock/setup.py b/trustgraph-bedrock/setup.py new file mode 100644 index 00000000..317f2bdf --- /dev/null +++ b/trustgraph-bedrock/setup.py @@ -0,0 +1,45 @@ +import setuptools +import os +import importlib + +with open("README.md", "r") as fh: + long_description = fh.read() + +# Load a version number module +spec = importlib.util.spec_from_file_location( + 'version', 'trustgraph/bedrock_version.py' +) +version_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(version_module) + +version = version_module.__version__ + +setuptools.setup( + name="trustgraph-bedrock", + version=version, + author="trustgraph.ai", + author_email="security@trustgraph.ai", + description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/trustgraph-ai/trustgraph", + packages=setuptools.find_namespace_packages( + where='./', + ), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Operating System :: OS Independent", + ], + python_requires='>=3.8', + download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", + install_requires=[ + "trustgraph-base", + "pulsar-client", + "prometheus-client", + "boto3", + ], + scripts=[ + "scripts/text-completion-bedrock", + ] +) diff --git a/trustgraph-flow/trustgraph/model/text_completion/bedrock/__init__.py b/trustgraph-bedrock/trustgraph/model/text_completion/bedrock/__init__.py similarity index 100% rename from trustgraph-flow/trustgraph/model/text_completion/bedrock/__init__.py rename to trustgraph-bedrock/trustgraph/model/text_completion/bedrock/__init__.py diff --git a/trustgraph-flow/trustgraph/model/text_completion/bedrock/__main__.py b/trustgraph-bedrock/trustgraph/model/text_completion/bedrock/__main__.py similarity index 100% rename from trustgraph-flow/trustgraph/model/text_completion/bedrock/__main__.py rename to trustgraph-bedrock/trustgraph/model/text_completion/bedrock/__main__.py diff --git a/trustgraph-flow/trustgraph/model/text_completion/bedrock/llm.py b/trustgraph-bedrock/trustgraph/model/text_completion/bedrock/llm.py similarity index 100% rename from trustgraph-flow/trustgraph/model/text_completion/bedrock/llm.py rename to trustgraph-bedrock/trustgraph/model/text_completion/bedrock/llm.py diff --git a/trustgraph-flow/setup.py b/trustgraph-flow/setup.py index 02578a4c..3b66d01d 100644 --- a/trustgraph-flow/setup.py +++ b/trustgraph-flow/setup.py @@ -49,12 +49,10 @@ setuptools.setup( "qdrant-client", "tabulate", "anthropic", - "google-cloud-aiplatform", "pyyaml", "prometheus-client", "pyarrow", "cohere", - "boto3", "openai", "neo4j", "tiktoken", @@ -98,13 +96,11 @@ setuptools.setup( "scripts/rows-write-cassandra", "scripts/run-processing", "scripts/text-completion-azure", - "scripts/text-completion-bedrock", "scripts/text-completion-claude", "scripts/text-completion-cohere", "scripts/text-completion-llamafile", "scripts/text-completion-ollama", "scripts/text-completion-openai", - "scripts/text-completion-vertexai", "scripts/tg-init-pulsar", "scripts/tg-processor-state", "scripts/triples-dump-parquet", diff --git a/trustgraph-utils/setup.py b/trustgraph-utils/setup.py new file mode 100644 index 00000000..3965b4f0 --- /dev/null +++ b/trustgraph-utils/setup.py @@ -0,0 +1,108 @@ +import setuptools +import os + +with open("README.md", "r") as fh: + long_description = fh.read() + +version = "0.11.7" + +setuptools.setup( + name="trustgraph-core", + version=version, + author="trustgraph.ai", + author_email="security@trustgraph.ai", + description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/trustgraph-ai/trustgraph", + packages=setuptools.find_namespace_packages( + where='./', +# include=['trustgraph.core'] + ), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Operating System :: OS Independent", + ], + python_requires='>=3.8', + download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", + install_requires=[ + "urllib3", + "rdflib", + "pymilvus", + "langchain", + "langchain-core", + "langchain-text-splitters", + "langchain-community", + "requests", + "cassandra-driver", + "pulsar-client", + "pypdf", + "qdrant-client", + "tabulate", + "anthropic", + "google-cloud-aiplatform", + "pyyaml", + "prometheus-client", + "pyarrow", + "cohere", + "boto3", + "openai", + "neo4j", + "tiktoken", + ], + scripts=[ + "scripts/chunker-recursive", + "scripts/chunker-token", + "scripts/concat-parquet", + "scripts/de-query-milvus", + "scripts/de-query-qdrant", + "scripts/de-write-milvus", + "scripts/de-write-qdrant", + "scripts/document-rag", + "scripts/dump-parquet", + "scripts/embeddings-ollama", + "scripts/embeddings-vectorize", + "scripts/ge-dump-parquet", + "scripts/ge-query-milvus", + "scripts/ge-query-qdrant", + "scripts/ge-write-milvus", + "scripts/ge-write-qdrant", + "scripts/graph-rag", + "scripts/graph-show", + "scripts/graph-to-turtle", + "scripts/init-pulsar-manager", + "scripts/kg-extract-definitions", + "scripts/kg-extract-topics", + "scripts/kg-extract-relationships", + "scripts/load-graph-embeddings", + "scripts/load-pdf", + "scripts/load-text", + "scripts/load-triples", + "scripts/metering", + "scripts/object-extract-row", + "scripts/oe-write-milvus", + "scripts/pdf-decoder", + "scripts/prompt-generic", + "scripts/prompt-template", + "scripts/query-document-rag", + "scripts/query-graph-rag", + "scripts/rows-write-cassandra", + "scripts/run-processing", + "scripts/text-completion-azure", + "scripts/text-completion-bedrock", + "scripts/text-completion-claude", + "scripts/text-completion-cohere", + "scripts/text-completion-llamafile", + "scripts/text-completion-ollama", + "scripts/text-completion-openai", + "scripts/text-completion-vertexai", + "scripts/tg-init-pulsar", + "scripts/tg-processor-state", + "scripts/triples-dump-parquet", + "scripts/triples-query-cassandra", + "scripts/triples-query-neo4j", + "scripts/triples-write-cassandra", + "scripts/triples-write-neo4j", + ] +) diff --git a/trustgraph-flow/scripts/text-completion-vertexai b/trustgraph-vertexai/scripts/text-completion-vertexai similarity index 100% rename from trustgraph-flow/scripts/text-completion-vertexai rename to trustgraph-vertexai/scripts/text-completion-vertexai diff --git a/trustgraph-vertexai/setup.py b/trustgraph-vertexai/setup.py new file mode 100644 index 00000000..73c03b20 --- /dev/null +++ b/trustgraph-vertexai/setup.py @@ -0,0 +1,45 @@ +import setuptools +import os +import importlib + +with open("README.md", "r") as fh: + long_description = fh.read() + +# Load a version number module +spec = importlib.util.spec_from_file_location( + 'version', 'trustgraph/vertexai_version.py' +) +version_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(version_module) + +version = version_module.__version__ + +setuptools.setup( + name="trustgraph-vertexai", + version=version, + author="trustgraph.ai", + author_email="security@trustgraph.ai", + description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/trustgraph-ai/trustgraph", + packages=setuptools.find_namespace_packages( + where='./', + ), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Operating System :: OS Independent", + ], + python_requires='>=3.8', + download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", + install_requires=[ + "trustgraph-base", + "pulsar-client", + "google-cloud-aiplatform", + "prometheus-client", + ], + scripts=[ + "scripts/text-completion-vertexai", + ] +) diff --git a/trustgraph-flow/trustgraph/model/text_completion/vertexai/__init__.py b/trustgraph-vertexai/trustgraph/model/text-completion/vertexai/__init__.py similarity index 100% rename from trustgraph-flow/trustgraph/model/text_completion/vertexai/__init__.py rename to trustgraph-vertexai/trustgraph/model/text-completion/vertexai/__init__.py diff --git a/trustgraph-flow/trustgraph/model/text_completion/vertexai/__main__.py b/trustgraph-vertexai/trustgraph/model/text-completion/vertexai/__main__.py similarity index 100% rename from trustgraph-flow/trustgraph/model/text_completion/vertexai/__main__.py rename to trustgraph-vertexai/trustgraph/model/text-completion/vertexai/__main__.py diff --git a/trustgraph-flow/trustgraph/model/text_completion/vertexai/llm.py b/trustgraph-vertexai/trustgraph/model/text-completion/vertexai/llm.py similarity index 100% rename from trustgraph-flow/trustgraph/model/text_completion/vertexai/llm.py rename to trustgraph-vertexai/trustgraph/model/text-completion/vertexai/llm.py