More package hacking

Containerfile

@@ -28,17 +28,19 @@ RUN pip3 install anthropic boto3 cohere openai google-cloud-aiplatform ollama \
 
 FROM ai AS build
 
-env PACKAGE_VERSION=0.0.0
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY README.md /root/build/trustgraph-base/
 
-COPY trustgraph-core/ /root/build/trustgraph-core/
-COPY README.md /root/build/trustgraph-core/
+COPY trustgraph-flow/ /root/build/trustgraph-flow/
+COPY README.md /root/build/trustgraph-flow/
 
 COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/
 COPY README.md /root/build/trustgraph-embeddings-hf/
 
 WORKDIR /root/build/
 
-RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-core/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
 RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/
 
 RUN ls /root/wheels
@@ -52,7 +54,8 @@ FROM ai
 COPY --from=build /root/wheels /root/wheels
 
 RUN \
-    pip3 install /root/wheels/trustgraph_core-* && \
+    pip3 install /root/wheels/trustgraph_base-* && \
+    pip3 install /root/wheels/trustgraph_flow-* && \
     pip3 install /root/wheels/trustgraph_embeddings_hf-* && \
     pip3 cache purge && \
     rm -rf /root/wheels

Makefile

@@ -1,19 +1,21 @@
 
 # VERSION=$(shell git describe | sed 's/^v//')
-VERSION=0.11.8
+VERSION=0.11.9
 
 DOCKER=podman
 
 all: container
 
 pkgs:
-	pip3 wheel --no-deps --wheel-dir dist trustgraph-core/
+	pip3 wheel --no-deps --wheel-dir dist trustgraph-base/
+	pip3 wheel --no-deps --wheel-dir dist trustgraph-flow/
 	pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/
 
 CONTAINER=docker.io/trustgraph/trustgraph-flow
 
 update-package-versions:
-	echo __version__ = \"${VERSION}\" > trustgraph-core/trustgraph/core_version.py
+	echo __version__ = \"${VERSION}\" > trustgraph-base/trustgraph/base_version.py
+	echo __version__ = \"${VERSION}\" > trustgraph-flow/trustgraph/flow_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py
 
 container: update-package-versions
@@ -23,17 +25,6 @@ container: update-package-versions
 push:
 	${DOCKER} push ${CONTAINER}:${VERSION}
 
-start:
-	${DOCKER} run -i -t --name ${NAME} \
-	    -i -t \
-	    -p 8081:8081 \
-	    -v $$(pwd)/keys:/keys \
-	    -v $$(pwd)/configs:/configs \
-	    ${CONTAINER}:${VERSION}
-
-stop:
-	${DOCKER} rm -f ${NAME}
-
 clean:
 	rm -rf wheels/
 
@@ -61,10 +52,8 @@ JSONNET_FLAGS=-J templates -J .
 # is going away in 0.11.
 
 update-templates: update-dcs
-# update-minikubes
 
 JSON_TO_YAML=python3 -c 'import sys, yaml, json; j=json.loads(sys.stdin.read()); print(yaml.safe_dump(j))'
-# JSON_TO_YAML=cat
 
 update-dcs: set-version
 	for graph in ${GRAPHS}; do \
@@ -88,31 +77,7 @@ update-dcs: set-version
 	  done; \
 	done
 
-update-minikubes: set-version
-	rm -rf deploy/minikube
-	mkdir -p deploy/minikube
-	for model in ${MODELS}; do \
-	  for graph in ${GRAPHS}; do \
-	    cm=$${graph},pulsar,${VECTORDB},embeddings-hf,graph-rag,grafana,trustgraph,$${model}; \
-	    input=templates/opts-to-minikube-k8s.jsonnet; \
-	    output=deploy/minikube/tg-launch-$${model}-$${graph}.yaml; \
-	    echo $${model} + $${graph} '->' $${output}; \
-	    jsonnet ${JSONNET_FLAGS} \
-	        --ext-str options=$${cm} $${input} | \
-	        ${JSON_TO_YAML} > $${output}; \
-	  done; \
-	done
-
 docker-hub-login:
 	cat docker-token.txt | \
 	    docker login -u trustgraph --password-stdin registry-1.docker.io
 
-FORCE:
-
-IGNOREconfig.yaml: config.json FORCE
-	jsonnet -J . -J templates/ templates/config-to-gcp-k8s.jsonnet | \
-	    python3 -c 'import sys, yaml, json; j=json.loads(sys.stdin.read()); print(yaml.safe_dump(j))' > $@
-
-config.yaml: config.json FORCE
-	jsonnet -J . -J templates/ templates/config-to-minikube-k8s.jsonnet | \
-	    python3 -c 'import sys, yaml, json; j=json.loads(sys.stdin.read()); print(yaml.safe_dump(j))' > $@

trustgraph-base/setup.py

@@ -7,7 +7,7 @@ with open("README.md", "r") as fh:
 
 # Load a version number module
 spec = importlib.util.spec_from_file_location(
-    'version', 'trustgraph/core_version.py'
+    'version', 'trustgraph/base_version.py'
 )
 version_module = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(version_module)
@@ -15,7 +15,7 @@ spec.loader.exec_module(version_module)
 version = version_module.__version__
 
 setuptools.setup(
-    name="trustgraph-core",
+    name="trustgraph-base",
     version=version,
     author="trustgraph.ai",
     author_email="security@trustgraph.ai",
@@ -25,7 +25,6 @@ setuptools.setup(
     url="https://github.com/trustgraph-ai/trustgraph",
     packages=setuptools.find_namespace_packages(
         where='./',
-#         include=['trustgraph.core']
     ),
     classifiers=[ 
         "Programming Language :: Python :: 3",
@@ -35,82 +34,9 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "urllib3",
-        "rdflib",
-        "pymilvus",
-        "langchain",
-        "langchain-core",
-        "langchain-text-splitters",
-        "langchain-community",
-        "requests",
-        "cassandra-driver",
         "pulsar-client",
-        "pypdf",
-        "qdrant-client",
-        "tabulate",
-        "anthropic",
-        "google-cloud-aiplatform",
-        "pyyaml",
         "prometheus-client",
-        "pyarrow",
-        "cohere",
-        "boto3",
-        "openai",
-        "neo4j",
-        "tiktoken",
     ],
     scripts=[
-        "scripts/chunker-recursive",
-        "scripts/chunker-token",
-        "scripts/concat-parquet",
-        "scripts/de-query-milvus",
-        "scripts/de-query-qdrant",
-        "scripts/de-write-milvus",
-        "scripts/de-write-qdrant",
-        "scripts/document-rag",
-        "scripts/dump-parquet",
-        "scripts/embeddings-ollama",
-        "scripts/embeddings-vectorize",
-        "scripts/ge-dump-parquet",
-        "scripts/ge-query-milvus",
-        "scripts/ge-query-qdrant",
-        "scripts/ge-write-milvus",
-        "scripts/ge-write-qdrant",
-        "scripts/graph-rag",
-        "scripts/graph-show",
-        "scripts/graph-to-turtle",
-        "scripts/init-pulsar-manager",
-        "scripts/kg-extract-definitions",
-        "scripts/kg-extract-topics",
-        "scripts/kg-extract-relationships",
-        "scripts/load-graph-embeddings",
-        "scripts/load-pdf",
-        "scripts/load-text",
-        "scripts/load-triples",
-        "scripts/metering",
-        "scripts/object-extract-row",
-        "scripts/oe-write-milvus",
-        "scripts/pdf-decoder",
-        "scripts/prompt-generic",
-        "scripts/prompt-template",
-        "scripts/query-document-rag",
-        "scripts/query-graph-rag",
-        "scripts/rows-write-cassandra",
-        "scripts/run-processing",
-        "scripts/text-completion-azure",
-        "scripts/text-completion-bedrock",
-        "scripts/text-completion-claude",
-        "scripts/text-completion-cohere",
-        "scripts/text-completion-llamafile",
-        "scripts/text-completion-ollama",
-        "scripts/text-completion-openai",
-        "scripts/text-completion-vertexai",
-        "scripts/tg-init-pulsar",
-        "scripts/tg-processor-state",
-        "scripts/triples-dump-parquet",
-        "scripts/triples-query-cassandra",
-        "scripts/triples-query-neo4j",
-        "scripts/triples-write-cassandra",
-        "scripts/triples-write-neo4j",
     ]
 )

trustgraph-embeddings-hf/setup.py

@@ -25,7 +25,6 @@ setuptools.setup(
     url="https://github.com/trustgraph-ai/trustgraph",
     packages=setuptools.find_namespace_packages(
         where='./',
-#         include=['trustgraph.core']
     ),
     classifiers=[ 
         "Programming Language :: Python :: 3",
@@ -35,7 +34,8 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
-        "trustgraph-core",
+        "trustgraph-base",
+        "trustgraph-flow",
         "torch",
         "urllib3",
         "transformers",

trustgraph-flow/setup.py

@@ -7,7 +7,7 @@ with open("README.md", "r") as fh:
 
 # Load a version number module
 spec = importlib.util.spec_from_file_location(
-    'version', 'trustgraph/core_version.py'
+    'version', 'trustgraph/flow_version.py'
 )
 version_module = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(version_module)
@@ -15,7 +15,7 @@ spec.loader.exec_module(version_module)
 version = version_module.__version__
 
 setuptools.setup(
-    name="trustgraph-core",
+    name="trustgraph-flow",
     version=version,
     author="trustgraph.ai",
     author_email="security@trustgraph.ai",
@@ -25,7 +25,6 @@ setuptools.setup(
     url="https://github.com/trustgraph-ai/trustgraph",
     packages=setuptools.find_namespace_packages(
         where='./',
-#         include=['trustgraph.core']
     ),
     classifiers=[ 
         "Programming Language :: Python :: 3",
@@ -35,6 +34,7 @@ setuptools.setup(
     python_requires='>=3.8',
     download_url = "https://github.com/trustgraph-ai/trustgraph/archive/refs/tags/v" + version + ".tar.gz",
     install_requires=[
+        "trustgraph-base",
         "urllib3",
         "rdflib",
         "pymilvus",