From 2fba68843560f35aab4135a68ee27d81083a6c35 Mon Sep 17 00:00:00 2001 From: Cyber MacGeddon Date: Tue, 28 Jan 2025 19:13:39 +0000 Subject: [PATCH] Separate containerfiles --- Makefile | 14 ++++-- containers/Containerfile.base | 48 ++++++++++++++++++++ containers/Containerfile.bedrock | 48 ++++++++++++++++++++ containers/Containerfile.flow | 60 +++++++++++++++++++++++++ containers/Containerfile.hf | 75 +++++++++++++++++++++++++++++++ containers/Containerfile.vertexai | 51 +++++++++++++++++++++ 6 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 containers/Containerfile.base create mode 100644 containers/Containerfile.bedrock create mode 100644 containers/Containerfile.flow create mode 100644 containers/Containerfile.hf create mode 100644 containers/Containerfile.vertexai diff --git a/Makefile b/Makefile index f408a31f..b78a54f3 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ packages: update-package-versions pypi-upload: twine upload dist/*-${VERSION}.* -CONTAINER=docker.io/trustgraph/trustgraph-flow +CONTAINER_BASE=docker.io/trustgraph update-package-versions: mkdir -p trustgraph-cli/trustgraph @@ -44,8 +44,16 @@ update-package-versions: echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py container: update-package-versions - ${DOCKER} build -f Containerfile -t ${CONTAINER}:${VERSION} \ - --format docker + ${DOCKER} build -f containers/Containerfile.base \ + -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} . + ${DOCKER} build -f containers/Containerfile.flow \ + -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} . + ${DOCKER} build -f containers/Containerfile.bedrock \ + -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} . + ${DOCKER} build -f containers/Containerfile.vertexai \ + -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} . + ${DOCKER} build -f containers/Containerfile.hf \ + -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} . push: ${DOCKER} push ${CONTAINER}:${VERSION} diff --git a/containers/Containerfile.base b/containers/Containerfile.base new file mode 100644 index 00000000..b4f5bbbf --- /dev/null +++ b/containers/Containerfile.base @@ -0,0 +1,48 @@ + +# ---------------------------------------------------------------------------- +# Build an AI container. This does the torch install which is huge, and I +# like to avoid re-doing this. +# ---------------------------------------------------------------------------- + +FROM docker.io/fedora:40 AS base + +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp && \ + dnf clean all + +RUN pip3 install --no-cache-dir pulsar-client==3.5.0 + +# ---------------------------------------------------------------------------- +# Build a container which contains the built Python packages. The build +# creates a bunch of left-over cruft, a separate phase means this is only +# needed to support package build +# ---------------------------------------------------------------------------- + +FROM base AS build + +COPY trustgraph-base/ /root/build/trustgraph-base/ +COPY trustgraph-cli/ /root/build/trustgraph-cli/ + +WORKDIR /root/build/ + +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/ + +# ---------------------------------------------------------------------------- +# Finally, the target container. Start with base and add the package. +# ---------------------------------------------------------------------------- + +FROM base + +COPY --from=build /root/wheels /root/wheels + +RUN \ + pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \ + pip3 install --no-cache-dir /root/wheels/trustgraph_cli-* && \ + rm -rf /root/wheels + +WORKDIR / + + + diff --git a/containers/Containerfile.bedrock b/containers/Containerfile.bedrock new file mode 100644 index 00000000..21819973 --- /dev/null +++ b/containers/Containerfile.bedrock @@ -0,0 +1,48 @@ + +# ---------------------------------------------------------------------------- +# Build an AI container. This does the torch install which is huge, and I +# like to avoid re-doing this. +# ---------------------------------------------------------------------------- + +FROM docker.io/fedora:40 AS base + +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \ + python3-rdflib + +RUN pip3 install --no-cache-dir boto3 pulsar-client==3.5.0 + +# ---------------------------------------------------------------------------- +# Build a container which contains the built Python packages. The build +# creates a bunch of left-over cruft, a separate phase means this is only +# needed to support package build +# ---------------------------------------------------------------------------- + +FROM base AS build + +COPY trustgraph-base/ /root/build/trustgraph-base/ +COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/ + +WORKDIR /root/build/ + +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/ + +RUN ls /root/wheels + +# ---------------------------------------------------------------------------- +# Finally, the target container. Start with base and add the package. +# ---------------------------------------------------------------------------- + +FROM base + +COPY --from=build /root/wheels /root/wheels + +RUN \ + pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \ + pip3 install --no-cache-dir /root/wheels/trustgraph_bedrock-* && \ + rm -rf /root/wheels + +WORKDIR / + diff --git a/containers/Containerfile.flow b/containers/Containerfile.flow new file mode 100644 index 00000000..8d47effe --- /dev/null +++ b/containers/Containerfile.flow @@ -0,0 +1,60 @@ + +# ---------------------------------------------------------------------------- +# Build an AI container. This does the torch install which is huge, and I +# like to avoid re-doing this. +# ---------------------------------------------------------------------------- + +FROM docker.io/fedora:40 AS base + +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \ + python3-rdflib + +RUN pip3 install --no-cache-dir \ + anthropic cohere openai google-generativeai \ + ollama \ + langchain==0.3.13 langchain-core==0.3.28 \ + langchain-text-splitters==0.3.4 \ + langchain-community==0.3.13 \ + pymilvus \ + pulsar-client==3.5.0 cassandra-driver pyyaml \ + neo4j tiktoken falkordb && \ + pip3 cache purge + +# ---------------------------------------------------------------------------- +# Build a container which contains the built Python packages. The build +# creates a bunch of left-over cruft, a separate phase means this is only +# needed to support package build +# ---------------------------------------------------------------------------- + +FROM base AS build + +COPY trustgraph-base/ /root/build/trustgraph-base/ +COPY trustgraph-flow/ /root/build/trustgraph-flow/ +COPY trustgraph-cli/ /root/build/trustgraph-cli/ + +WORKDIR /root/build/ + +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/ + +RUN ls /root/wheels + +# ---------------------------------------------------------------------------- +# Finally, the target container. Start with base and add the package. +# ---------------------------------------------------------------------------- + +FROM base + +COPY --from=build /root/wheels /root/wheels + +RUN \ + pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \ + pip3 install --no-cache-dir /root/wheels/trustgraph_flow-* && \ + pip3 install --no-cache-dir /root/wheels/trustgraph_cli-* && \ + rm -rf /root/wheels + +WORKDIR / + diff --git a/containers/Containerfile.hf b/containers/Containerfile.hf new file mode 100644 index 00000000..4076db28 --- /dev/null +++ b/containers/Containerfile.hf @@ -0,0 +1,75 @@ + +# ---------------------------------------------------------------------------- +# Build an AI container. This does the torch install which is huge, and I +# like to avoid re-doing this. +# ---------------------------------------------------------------------------- + +FROM docker.io/fedora:40 AS ai + +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \ + python3-rdflib + +RUN pip3 install torch==2.5.1+cpu \ + --index-url https://download.pytorch.org/whl/cpu + +RUN pip3 install --no-cache-dir \ + langchain==0.3.13 langchain-core==0.3.28 langchain-huggingface==0.1.2 \ + langchain-community==0.3.13 \ + sentence-transformers==3.4.0 transformers==4.47.1 \ + huggingface-hub==0.27.0 \ + pulsar-client==3.5.0 + +# Most commonly used embeddings model, just build it into the container +# image +RUN huggingface-cli download sentence-transformers/all-MiniLM-L6-v2 + +# ---------------------------------------------------------------------------- +# Build a container which contains the built Python packages. The build +# creates a bunch of left-over cruft, a separate phase means this is only +# needed to support package build +# ---------------------------------------------------------------------------- + +FROM ai AS build + +COPY trustgraph-base/ /root/build/trustgraph-base/ +COPY trustgraph-flow/ /root/build/trustgraph-flow/ +COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/ +COPY trustgraph-bedrock/ /root/build/trustgraph-bedrock/ +COPY trustgraph-embeddings-hf/ /root/build/trustgraph-embeddings-hf/ +COPY trustgraph-cli/ /root/build/trustgraph-cli/ + +WORKDIR /root/build/ + +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-flow/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-bedrock/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-embeddings-hf/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-cli/ + +RUN ls /root/wheels + +# ---------------------------------------------------------------------------- +# Finally, the target container. Start with base and add the package. +# ---------------------------------------------------------------------------- + +FROM ai + +COPY --from=build /root/wheels /root/wheels + +RUN \ + pip3 install /root/wheels/trustgraph_base-* && \ + pip3 install /root/wheels/trustgraph_flow-* && \ + pip3 install /root/wheels/trustgraph_vertexai-* && \ + pip3 install /root/wheels/trustgraph_bedrock-* && \ + pip3 install /root/wheels/trustgraph_embeddings_hf-* && \ + pip3 install /root/wheels/trustgraph_cli-* && \ + pip3 cache purge && \ + rm -rf /root/wheels + +WORKDIR / + +CMD sleep 1000000 + diff --git a/containers/Containerfile.vertexai b/containers/Containerfile.vertexai new file mode 100644 index 00000000..72d21bde --- /dev/null +++ b/containers/Containerfile.vertexai @@ -0,0 +1,51 @@ + +# ---------------------------------------------------------------------------- +# Build an AI container. This does the torch install which is huge, and I +# like to avoid re-doing this. +# ---------------------------------------------------------------------------- + +FROM docker.io/fedora:40 AS base + +ENV PIP_BREAK_SYSTEM_PACKAGES=1 + +RUN dnf install -y python3 python3-pip python3-wheel python3-aiohttp \ + python3-rdflib + +RUN pip3 install --no-cache-dir \ + google-cloud-aiplatform pulsar-client==3.5.0 + +# ---------------------------------------------------------------------------- +# Build a container which contains the built Python packages. The build +# creates a bunch of left-over cruft, a separate phase means this is only +# needed to support package build +# ---------------------------------------------------------------------------- + +FROM base AS build + +COPY trustgraph-base/ /root/build/trustgraph-base/ +COPY trustgraph-vertexai/ /root/build/trustgraph-vertexai/ + +WORKDIR /root/build/ + +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/ +RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-vertexai/ + +RUN ls /root/wheels + +# ---------------------------------------------------------------------------- +# Finally, the target container. Start with base and add the package. +# ---------------------------------------------------------------------------- + +FROM base + +COPY --from=build /root/wheels /root/wheels + +RUN \ + pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \ + pip3 install --no-cache-dir /root/wheels/trustgraph_vertexai-* && \ + rm -rf /root/wheels + +WORKDIR / + + +