Separate containerfiles

Makefile

@@ -30,7 +30,7 @@ packages: update-package-versions
 pypi-upload:
 	twine upload dist/*-${VERSION}.*
 
-CONTAINER=docker.io/trustgraph/trustgraph-flow
+CONTAINER_BASE=docker.io/trustgraph
 
 update-package-versions:
 	mkdir -p trustgraph-cli/trustgraph
@@ -44,8 +44,16 @@ update-package-versions:
 	echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
 
 container: update-package-versions
-	${DOCKER} build -f Containerfile -t ${CONTAINER}:${VERSION} \
-	    --format docker
+	${DOCKER} build -f containers/Containerfile.base \
+	    -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.flow \
+	    -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.bedrock \
+	    -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.vertexai \
+	    -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
+	${DOCKER} build -f containers/Containerfile.hf \
+	    -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
 
 push:
 	${DOCKER} push ${CONTAINER}:${VERSION}

containers/Containerfile.base

@@ -0,0 +1,48 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp && \
+    dnf clean all
+
+RUN pip3 install --no-cache-dir pulsar-client==3.5.0
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-cli/ /root/build/trustgraph-cli/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_cli-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+
+
+

containers/Containerfile.bedrock

@@ -0,0 +1,48 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install --no-cache-dir boto3 pulsar-client==3.5.0
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_bedrock-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+

containers/Containerfile.flow

@@ -0,0 +1,60 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install --no-cache-dir \
+    anthropic cohere openai google-generativeai \
+    ollama \
+    langchain==0.3.13 langchain-core==0.3.28 \
+    langchain-text-splitters==0.3.4 \
+    langchain-community==0.3.13 \
+    pymilvus \
+    pulsar-client==3.5.0 cassandra-driver pyyaml \
+    neo4j tiktoken falkordb && \
+    pip3 cache purge
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-flow/ /root/build/trustgraph-flow/
+COPY trustgraph-cli/ /root/build/trustgraph-cli/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_flow-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_cli-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+

containers/Containerfile.hf

@@ -0,0 +1,75 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS ai
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install torch==2.5.1+cpu \
+    --index-url https://download.pytorch.org/whl/cpu
+
+RUN pip3 install --no-cache-dir \
+    langchain==0.3.13 langchain-core==0.3.28 langchain-huggingface==0.1.2 \
+    langchain-community==0.3.13 \
+    sentence-transformers==3.4.0 transformers==4.47.1 \
+    huggingface-hub==0.27.0 \
+    pulsar-client==3.5.0
+
+# Most commonly used embeddings model, just build it into the container
+# image
+RUN huggingface-cli download sentence-transformers/all-MiniLM-L6-v2
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM ai AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-flow/ /root/build/trustgraph-flow/
+COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/
+COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/
+COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/
+COPY trustgraph-cli/ /root/build/trustgraph-cli/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM ai
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install /root/wheels/trustgraph_base-* && \
+    pip3 install /root/wheels/trustgraph_flow-* && \
+    pip3 install /root/wheels/trustgraph_vertexai-* && \
+    pip3 install /root/wheels/trustgraph_bedrock-* && \
+    pip3 install /root/wheels/trustgraph_embeddings_hf-* && \
+    pip3 install /root/wheels/trustgraph_cli-* && \
+    pip3 cache purge && \
+    rm -rf /root/wheels
+
+WORKDIR /
+
+CMD sleep 1000000
+

containers/Containerfile.vertexai

@@ -0,0 +1,51 @@
+
+# ----------------------------------------------------------------------------
+# Build an AI container.  This does the torch install which is huge, and I
+# like to avoid re-doing this.
+# ----------------------------------------------------------------------------
+
+FROM docker.io/fedora:40 AS base
+
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+
+RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \
+    python3-rdflib
+
+RUN pip3 install --no-cache-dir \
+    google-cloud-aiplatform pulsar-client==3.5.0
+
+# ----------------------------------------------------------------------------
+# Build a container which contains the built Python packages.  The build
+# creates a bunch of left-over cruft, a separate phase means this is only
+# needed to support package build
+# ----------------------------------------------------------------------------
+
+FROM base AS build
+
+COPY trustgraph-base/ /root/build/trustgraph-base/
+COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/
+
+WORKDIR /root/build/
+
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
+RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/
+
+RUN ls /root/wheels
+
+# ----------------------------------------------------------------------------
+# Finally, the target container.  Start with base and add the package.
+# ----------------------------------------------------------------------------
+
+FROM base
+
+COPY --from=build /root/wheels /root/wheels
+
+RUN \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
+    pip3 install --no-cache-dir /root/wheels/trustgraph_vertexai-* && \
+    rm -rf /root/wheels
+
+WORKDIR /
+
+
+