From 256c115bde4a9d099cf484b83281d14ece0f61a8 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Mon, 30 Sep 2024 17:22:48 +0100 Subject: [PATCH] Starting to spawn base package --- trustgraph-base/setup.py | 116 ++++++++++++++++++ .../trustgraph/base/__init__.py | 0 .../trustgraph/base/base_processor.py | 0 .../trustgraph/base/consumer.py | 0 .../trustgraph/base/consumer_producer.py | 0 .../trustgraph/base/producer.py | 0 .../trustgraph/clients/__init__.py | 0 .../trustgraph/clients/base.py | 0 .../clients/document_embeddings_client.py | 0 .../trustgraph/clients/document_rag_client.py | 0 .../trustgraph/clients/embeddings_client.py | 0 .../clients/graph_embeddings_client.py | 0 .../trustgraph/clients/graph_rag_client.py | 0 .../trustgraph/clients/llm_client.py | 0 .../trustgraph/clients/prompt_client.py | 0 .../clients/triples_query_client.py | 0 .../trustgraph/exceptions.py | 0 .../trustgraph/log_level.py | 0 .../trustgraph/objects/__init__.py | 0 .../trustgraph/objects/field.py | 0 .../trustgraph/objects/object.py | 0 .../trustgraph/rdf.py | 0 .../trustgraph/schema/__init__.py | 0 .../trustgraph/schema/documents.py | 0 .../trustgraph/schema/graph.py | 0 .../trustgraph/schema/models.py | 0 .../trustgraph/schema/object.py | 0 .../trustgraph/schema/prompt.py | 0 .../trustgraph/schema/retrieval.py | 0 .../trustgraph/schema/topic.py | 0 .../trustgraph/schema/types.py | 0 31 files changed, 116 insertions(+) create mode 100644 trustgraph-base/setup.py rename {trustgraph-core => trustgraph-base}/trustgraph/base/__init__.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/base/base_processor.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/base/consumer.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/base/consumer_producer.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/base/producer.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/clients/__init__.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/clients/base.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/clients/document_embeddings_client.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/clients/document_rag_client.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/clients/embeddings_client.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/clients/graph_embeddings_client.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/clients/graph_rag_client.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/clients/llm_client.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/clients/prompt_client.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/clients/triples_query_client.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/exceptions.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/log_level.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/objects/__init__.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/objects/field.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/objects/object.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/rdf.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/schema/__init__.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/schema/documents.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/schema/graph.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/schema/models.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/schema/object.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/schema/prompt.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/schema/retrieval.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/schema/topic.py (100%) rename {trustgraph-core => trustgraph-base}/trustgraph/schema/types.py (100%) diff --git a/trustgraph-base/setup.py b/trustgraph-base/setup.py new file mode 100644 index 00000000..404bb26b --- /dev/null +++ b/trustgraph-base/setup.py @@ -0,0 +1,116 @@ +import setuptools +import os +import importlib + +with open("README.md", "r") as fh: + long_description = fh.read() + +# Load a version number module +spec = importlib.util.spec_from_file_location( + 'version', 'trustgraph/core_version.py' +) +version_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(version_module) + +version = version_module.__version__ + +setuptools.setup( + name="trustgraph-core", + version=version, + author="trustgraph.ai", + author_email="security@trustgraph.ai", + description="TrustGraph provides a means to run a pipeline of flexible AI processing components in a flexible means to achieve a processing pipeline.", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/trustgraph-ai/trustgraph", + packages=setuptools.find_namespace_packages( + where='./', +# include=['trustgraph.core'] + ), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Operating System :: OS Independent", + ], + python_requires='>=3.8', + download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz", + install_requires=[ + "urllib3", + "rdflib", + "pymilvus", + "langchain", + "langchain-core", + "langchain-text-splitters", + "langchain-community", + "requests", + "cassandra-driver", + "pulsar-client", + "pypdf", + "qdrant-client", + "tabulate", + "anthropic", + "google-cloud-aiplatform", + "pyyaml", + "prometheus-client", + "pyarrow", + "cohere", + "boto3", + "openai", + "neo4j", + "tiktoken", + ], + scripts=[ + "scripts/chunker-recursive", + "scripts/chunker-token", + "scripts/concat-parquet", + "scripts/de-query-milvus", + "scripts/de-query-qdrant", + "scripts/de-write-milvus", + "scripts/de-write-qdrant", + "scripts/document-rag", + "scripts/dump-parquet", + "scripts/embeddings-ollama", + "scripts/embeddings-vectorize", + "scripts/ge-dump-parquet", + "scripts/ge-query-milvus", + "scripts/ge-query-qdrant", + "scripts/ge-write-milvus", + "scripts/ge-write-qdrant", + "scripts/graph-rag", + "scripts/graph-show", + "scripts/graph-to-turtle", + "scripts/init-pulsar-manager", + "scripts/kg-extract-definitions", + "scripts/kg-extract-topics", + "scripts/kg-extract-relationships", + "scripts/load-graph-embeddings", + "scripts/load-pdf", + "scripts/load-text", + "scripts/load-triples", + "scripts/metering", + "scripts/object-extract-row", + "scripts/oe-write-milvus", + "scripts/pdf-decoder", + "scripts/prompt-generic", + "scripts/prompt-template", + "scripts/query-document-rag", + "scripts/query-graph-rag", + "scripts/rows-write-cassandra", + "scripts/run-processing", + "scripts/text-completion-azure", + "scripts/text-completion-bedrock", + "scripts/text-completion-claude", + "scripts/text-completion-cohere", + "scripts/text-completion-llamafile", + "scripts/text-completion-ollama", + "scripts/text-completion-openai", + "scripts/text-completion-vertexai", + "scripts/tg-init-pulsar", + "scripts/tg-processor-state", + "scripts/triples-dump-parquet", + "scripts/triples-query-cassandra", + "scripts/triples-query-neo4j", + "scripts/triples-write-cassandra", + "scripts/triples-write-neo4j", + ] +) diff --git a/trustgraph-core/trustgraph/base/__init__.py b/trustgraph-base/trustgraph/base/__init__.py similarity index 100% rename from trustgraph-core/trustgraph/base/__init__.py rename to trustgraph-base/trustgraph/base/__init__.py diff --git a/trustgraph-core/trustgraph/base/base_processor.py b/trustgraph-base/trustgraph/base/base_processor.py similarity index 100% rename from trustgraph-core/trustgraph/base/base_processor.py rename to trustgraph-base/trustgraph/base/base_processor.py diff --git a/trustgraph-core/trustgraph/base/consumer.py b/trustgraph-base/trustgraph/base/consumer.py similarity index 100% rename from trustgraph-core/trustgraph/base/consumer.py rename to trustgraph-base/trustgraph/base/consumer.py diff --git a/trustgraph-core/trustgraph/base/consumer_producer.py b/trustgraph-base/trustgraph/base/consumer_producer.py similarity index 100% rename from trustgraph-core/trustgraph/base/consumer_producer.py rename to trustgraph-base/trustgraph/base/consumer_producer.py diff --git a/trustgraph-core/trustgraph/base/producer.py b/trustgraph-base/trustgraph/base/producer.py similarity index 100% rename from trustgraph-core/trustgraph/base/producer.py rename to trustgraph-base/trustgraph/base/producer.py diff --git a/trustgraph-core/trustgraph/clients/__init__.py b/trustgraph-base/trustgraph/clients/__init__.py similarity index 100% rename from trustgraph-core/trustgraph/clients/__init__.py rename to trustgraph-base/trustgraph/clients/__init__.py diff --git a/trustgraph-core/trustgraph/clients/base.py b/trustgraph-base/trustgraph/clients/base.py similarity index 100% rename from trustgraph-core/trustgraph/clients/base.py rename to trustgraph-base/trustgraph/clients/base.py diff --git a/trustgraph-core/trustgraph/clients/document_embeddings_client.py b/trustgraph-base/trustgraph/clients/document_embeddings_client.py similarity index 100% rename from trustgraph-core/trustgraph/clients/document_embeddings_client.py rename to trustgraph-base/trustgraph/clients/document_embeddings_client.py diff --git a/trustgraph-core/trustgraph/clients/document_rag_client.py b/trustgraph-base/trustgraph/clients/document_rag_client.py similarity index 100% rename from trustgraph-core/trustgraph/clients/document_rag_client.py rename to trustgraph-base/trustgraph/clients/document_rag_client.py diff --git a/trustgraph-core/trustgraph/clients/embeddings_client.py b/trustgraph-base/trustgraph/clients/embeddings_client.py similarity index 100% rename from trustgraph-core/trustgraph/clients/embeddings_client.py rename to trustgraph-base/trustgraph/clients/embeddings_client.py diff --git a/trustgraph-core/trustgraph/clients/graph_embeddings_client.py b/trustgraph-base/trustgraph/clients/graph_embeddings_client.py similarity index 100% rename from trustgraph-core/trustgraph/clients/graph_embeddings_client.py rename to trustgraph-base/trustgraph/clients/graph_embeddings_client.py diff --git a/trustgraph-core/trustgraph/clients/graph_rag_client.py b/trustgraph-base/trustgraph/clients/graph_rag_client.py similarity index 100% rename from trustgraph-core/trustgraph/clients/graph_rag_client.py rename to trustgraph-base/trustgraph/clients/graph_rag_client.py diff --git a/trustgraph-core/trustgraph/clients/llm_client.py b/trustgraph-base/trustgraph/clients/llm_client.py similarity index 100% rename from trustgraph-core/trustgraph/clients/llm_client.py rename to trustgraph-base/trustgraph/clients/llm_client.py diff --git a/trustgraph-core/trustgraph/clients/prompt_client.py b/trustgraph-base/trustgraph/clients/prompt_client.py similarity index 100% rename from trustgraph-core/trustgraph/clients/prompt_client.py rename to trustgraph-base/trustgraph/clients/prompt_client.py diff --git a/trustgraph-core/trustgraph/clients/triples_query_client.py b/trustgraph-base/trustgraph/clients/triples_query_client.py similarity index 100% rename from trustgraph-core/trustgraph/clients/triples_query_client.py rename to trustgraph-base/trustgraph/clients/triples_query_client.py diff --git a/trustgraph-core/trustgraph/exceptions.py b/trustgraph-base/trustgraph/exceptions.py similarity index 100% rename from trustgraph-core/trustgraph/exceptions.py rename to trustgraph-base/trustgraph/exceptions.py diff --git a/trustgraph-core/trustgraph/log_level.py b/trustgraph-base/trustgraph/log_level.py similarity index 100% rename from trustgraph-core/trustgraph/log_level.py rename to trustgraph-base/trustgraph/log_level.py diff --git a/trustgraph-core/trustgraph/objects/__init__.py b/trustgraph-base/trustgraph/objects/__init__.py similarity index 100% rename from trustgraph-core/trustgraph/objects/__init__.py rename to trustgraph-base/trustgraph/objects/__init__.py diff --git a/trustgraph-core/trustgraph/objects/field.py b/trustgraph-base/trustgraph/objects/field.py similarity index 100% rename from trustgraph-core/trustgraph/objects/field.py rename to trustgraph-base/trustgraph/objects/field.py diff --git a/trustgraph-core/trustgraph/objects/object.py b/trustgraph-base/trustgraph/objects/object.py similarity index 100% rename from trustgraph-core/trustgraph/objects/object.py rename to trustgraph-base/trustgraph/objects/object.py diff --git a/trustgraph-core/trustgraph/rdf.py b/trustgraph-base/trustgraph/rdf.py similarity index 100% rename from trustgraph-core/trustgraph/rdf.py rename to trustgraph-base/trustgraph/rdf.py diff --git a/trustgraph-core/trustgraph/schema/__init__.py b/trustgraph-base/trustgraph/schema/__init__.py similarity index 100% rename from trustgraph-core/trustgraph/schema/__init__.py rename to trustgraph-base/trustgraph/schema/__init__.py diff --git a/trustgraph-core/trustgraph/schema/documents.py b/trustgraph-base/trustgraph/schema/documents.py similarity index 100% rename from trustgraph-core/trustgraph/schema/documents.py rename to trustgraph-base/trustgraph/schema/documents.py diff --git a/trustgraph-core/trustgraph/schema/graph.py b/trustgraph-base/trustgraph/schema/graph.py similarity index 100% rename from trustgraph-core/trustgraph/schema/graph.py rename to trustgraph-base/trustgraph/schema/graph.py diff --git a/trustgraph-core/trustgraph/schema/models.py b/trustgraph-base/trustgraph/schema/models.py similarity index 100% rename from trustgraph-core/trustgraph/schema/models.py rename to trustgraph-base/trustgraph/schema/models.py diff --git a/trustgraph-core/trustgraph/schema/object.py b/trustgraph-base/trustgraph/schema/object.py similarity index 100% rename from trustgraph-core/trustgraph/schema/object.py rename to trustgraph-base/trustgraph/schema/object.py diff --git a/trustgraph-core/trustgraph/schema/prompt.py b/trustgraph-base/trustgraph/schema/prompt.py similarity index 100% rename from trustgraph-core/trustgraph/schema/prompt.py rename to trustgraph-base/trustgraph/schema/prompt.py diff --git a/trustgraph-core/trustgraph/schema/retrieval.py b/trustgraph-base/trustgraph/schema/retrieval.py similarity index 100% rename from trustgraph-core/trustgraph/schema/retrieval.py rename to trustgraph-base/trustgraph/schema/retrieval.py diff --git a/trustgraph-core/trustgraph/schema/topic.py b/trustgraph-base/trustgraph/schema/topic.py similarity index 100% rename from trustgraph-core/trustgraph/schema/topic.py rename to trustgraph-base/trustgraph/schema/topic.py diff --git a/trustgraph-core/trustgraph/schema/types.py b/trustgraph-base/trustgraph/schema/types.py similarity index 100% rename from trustgraph-core/trustgraph/schema/types.py rename to trustgraph-base/trustgraph/schema/types.py