apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-16 16:51:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Bedrock and VertexAI

Browse source
This commit is contained in:
Cyber MacGeddon 2024-09-30 17:42:59 +01:00
parent dc45babbb9
commit e0866d9f92
14 changed files with 212 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

10
Containerfile
View file

@ -34,6 +34,12 @@ COPY README.md /root/build/trustgraph-base/
COPY trustgraph-flow/ /root/build/trustgraph-flow/
COPY README.md /root/build/trustgraph-flow/
COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/
COPY README.md /root/build/trustgraph-vertexai/
COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
COPY README.md /root/build/trustgraph-bedrock/
COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/
COPY README.md /root/build/trustgraph-embeddings-hf/
@ -41,6 +47,8 @@ WORKDIR /root/build/
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/
RUN ls /root/wheels
@ -56,6 +64,8 @@ COPY --from=build /root/wheels /root/wheels
RUN \
pip3 install /root/wheels/trustgraph_base-* && \
pip3 install /root/wheels/trustgraph_flow-* && \
pip3 install /root/wheels/trustgraph_vertexai-* && \
pip3 install /root/wheels/trustgraph_bedrock-* && \
pip3 install /root/wheels/trustgraph_embeddings_hf-* && \
pip3 cache purge && \
rm -rf /root/wheels

4
Makefile
View file

@ -9,6 +9,8 @@ all: container
pkgs:
pip3 wheel --no-deps --wheel-dir dist trustgraph-base/
pip3 wheel --no-deps --wheel-dir dist trustgraph-flow/
pip3 wheel --no-deps --wheel-dir dist trustgraph-vertexai/
pip3 wheel --no-deps --wheel-dir dist trustgraph-bedrock/
pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/
CONTAINER=docker.io/trustgraph/trustgraph-flow
@ -16,6 +18,8 @@ CONTAINER=docker.io/trustgraph/trustgraph-flow
update-package-versions:
echo __version__ = \"${VERSION}\" > trustgraph-base/trustgraph/base_version.py
echo __version__ = \"${VERSION}\" > trustgraph-flow/trustgraph/flow_version.py
echo __version__ = \"${VERSION}\" > trustgraph-vertexai/trustgraph/vertexai_version.py
echo __version__ = \"${VERSION}\" > trustgraph-bedrock/trustgraph/bedrock_version.py
echo __version__ = \"${VERSION}\" > trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py
container: update-package-versions

0
trustgraph-flow/scripts/text-completion-bedrock → trustgraph-bedrock/scripts/text-completion-bedrock
View file

45
trustgraph-bedrock/setup.py Normal file
View file

@ -0,0 +1,45 @@
import setuptools
import os
import importlib
with open("README.md", "r") as fh:
long_description = fh.read()
# Load a version number module
spec = importlib.util.spec_from_file_location(
'version', 'trustgraph/bedrock_version.py'
)
version_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(version_module)
version = version_module.__version__
setuptools.setup(
name="trustgraph-bedrock",
version=version,
author="trustgraph.ai",
author_email="security@trustgraph.ai",
description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/trustgraph-ai/trustgraph",
packages=setuptools.find_namespace_packages(
where='./',
),
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
"Operating System :: OS Independent",
],
python_requires='>=3.8',
download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
install_requires=[
"trustgraph-base",
"pulsar-client",
"prometheus-client",
"boto3",
],
scripts=[
"scripts/text-completion-bedrock",
]
)

0
trustgraph-flow/trustgraph/model/text_completion/bedrock/__init__.py → trustgraph-bedrock/trustgraph/model/text_completion/bedrock/__init__.py
View file

0
trustgraph-flow/trustgraph/model/text_completion/bedrock/__main__.py → trustgraph-bedrock/trustgraph/model/text_completion/bedrock/__main__.py
View file

0
trustgraph-flow/trustgraph/model/text_completion/bedrock/llm.py → trustgraph-bedrock/trustgraph/model/text_completion/bedrock/llm.py
View file

4
trustgraph-flow/setup.py
View file

@ -49,12 +49,10 @@ setuptools.setup(
"qdrant-client",
"tabulate",
"anthropic",
"google-cloud-aiplatform",
"pyyaml",
"prometheus-client",
"pyarrow",
"cohere",
"boto3",
"openai",
"neo4j",
"tiktoken",
@ -98,13 +96,11 @@ setuptools.setup(
"scripts/rows-write-cassandra",
"scripts/run-processing",
"scripts/text-completion-azure",
"scripts/text-completion-bedrock",
"scripts/text-completion-claude",
"scripts/text-completion-cohere",
"scripts/text-completion-llamafile",
"scripts/text-completion-ollama",
"scripts/text-completion-openai",
"scripts/text-completion-vertexai",
"scripts/tg-init-pulsar",
"scripts/tg-processor-state",
"scripts/triples-dump-parquet",

108
trustgraph-utils/setup.py Normal file
View file

@ -0,0 +1,108 @@
import setuptools
import os
with open("README.md", "r") as fh:
long_description = fh.read()
version = "0.11.7"
setuptools.setup(
name="trustgraph-core",
version=version,
author="trustgraph.ai",
author_email="security@trustgraph.ai",
description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/trustgraph-ai/trustgraph",
packages=setuptools.find_namespace_packages(
where='./',
# include=['trustgraph.core']
),
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
"Operating System :: OS Independent",
],
python_requires='>=3.8',
download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
install_requires=[
"urllib3",
"rdflib",
"pymilvus",
"langchain",
"langchain-core",
"langchain-text-splitters",
"langchain-community",
"requests",
"cassandra-driver",
"pulsar-client",
"pypdf",
"qdrant-client",
"tabulate",
"anthropic",
"google-cloud-aiplatform",
"pyyaml",
"prometheus-client",
"pyarrow",
"cohere",
"boto3",
"openai",
"neo4j",
"tiktoken",
],
scripts=[
"scripts/chunker-recursive",
"scripts/chunker-token",
"scripts/concat-parquet",
"scripts/de-query-milvus",
"scripts/de-query-qdrant",
"scripts/de-write-milvus",
"scripts/de-write-qdrant",
"scripts/document-rag",
"scripts/dump-parquet",
"scripts/embeddings-ollama",
"scripts/embeddings-vectorize",
"scripts/ge-dump-parquet",
"scripts/ge-query-milvus",
"scripts/ge-query-qdrant",
"scripts/ge-write-milvus",
"scripts/ge-write-qdrant",
"scripts/graph-rag",
"scripts/graph-show",
"scripts/graph-to-turtle",
"scripts/init-pulsar-manager",
"scripts/kg-extract-definitions",
"scripts/kg-extract-topics",
"scripts/kg-extract-relationships",
"scripts/load-graph-embeddings",
"scripts/load-pdf",
"scripts/load-text",
"scripts/load-triples",
"scripts/metering",
"scripts/object-extract-row",
"scripts/oe-write-milvus",
"scripts/pdf-decoder",
"scripts/prompt-generic",
"scripts/prompt-template",
"scripts/query-document-rag",
"scripts/query-graph-rag",
"scripts/rows-write-cassandra",
"scripts/run-processing",
"scripts/text-completion-azure",
"scripts/text-completion-bedrock",
"scripts/text-completion-claude",
"scripts/text-completion-cohere",
"scripts/text-completion-llamafile",
"scripts/text-completion-ollama",
"scripts/text-completion-openai",
"scripts/text-completion-vertexai",
"scripts/tg-init-pulsar",
"scripts/tg-processor-state",
"scripts/triples-dump-parquet",
"scripts/triples-query-cassandra",
"scripts/triples-query-neo4j",
"scripts/triples-write-cassandra",
"scripts/triples-write-neo4j",
]
)

0
trustgraph-flow/scripts/text-completion-vertexai → trustgraph-vertexai/scripts/text-completion-vertexai
View file

45
trustgraph-vertexai/setup.py Normal file
View file

@ -0,0 +1,45 @@
import setuptools
import os
import importlib
with open("README.md", "r") as fh:
long_description = fh.read()
# Load a version number module
spec = importlib.util.spec_from_file_location(
'version', 'trustgraph/vertexai_version.py'
)
version_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(version_module)
version = version_module.__version__
setuptools.setup(
name="trustgraph-vertexai",
version=version,
author="trustgraph.ai",
author_email="security@trustgraph.ai",
description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/trustgraph-ai/trustgraph",
packages=setuptools.find_namespace_packages(
where='./',
),
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
"Operating System :: OS Independent",
],
python_requires='>=3.8',
download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
install_requires=[
"trustgraph-base",
"pulsar-client",
"google-cloud-aiplatform",
"prometheus-client",
],
scripts=[
"scripts/text-completion-vertexai",
]
)

0
trustgraph-flow/trustgraph/model/text_completion/vertexai/__init__.py → trustgraph-vertexai/trustgraph/model/text-completion/vertexai/__init__.py
View file

0
trustgraph-flow/trustgraph/model/text_completion/vertexai/__main__.py → trustgraph-vertexai/trustgraph/model/text-completion/vertexai/__main__.py
View file

0
trustgraph-flow/trustgraph/model/text_completion/vertexai/llm.py → trustgraph-vertexai/trustgraph/model/text-completion/vertexai/llm.py
View file