From 3c6adbf16a64e047985de7f1e4653fd7fe5e23b4 Mon Sep 17 00:00:00 2001 From: cybermaggedon Date: Mon, 13 Apr 2026 20:51:09 +0100 Subject: [PATCH] Use manifests to build for amd64 and arm64 (#798) Remove Pytorch +cpu invocation, so it builds on ARM --- .github/workflows/release.yaml | 25 +++--- Makefile | 160 ++++++--------------------------- containers/Containerfile.hf | 6 +- 3 files changed, 44 insertions(+), 147 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dc2fc89b..f29e642c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -52,20 +52,25 @@ jobs: strategy: matrix: container: - - trustgraph-base - - trustgraph-flow - - trustgraph-bedrock - - trustgraph-vertexai - - trustgraph-hf - - trustgraph-ocr - - trustgraph-unstructured - - trustgraph-mcp + - base + - flow + - bedrock + - vertexai + - hf + - ocr + - unstructured + - mcp steps: - name: Checkout uses: actions/checkout@v4 + - name: Set up QEMU - for ARM emulation + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - name: Docker Hub token run: echo ${{ secrets.DOCKER_SECRET }} > docker-token.txt @@ -80,8 +85,8 @@ jobs: run: make update-package-versions VERSION=${{ steps.version.outputs.VERSION }} - name: Build container - ${{ matrix.container }} - run: make container-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }} + run: make manifest-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }} - name: Push container - ${{ matrix.container }} - run: make push-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }} + run: make push-manifest-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }} diff --git a/Makefile b/Makefile index 197a6c63..593b7a9c 100644 --- a/Makefile +++ b/Makefile @@ -52,51 +52,12 @@ update-package-versions: echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py echo __version__ = \"${VERSION}\" > trustgraph-mcp/trustgraph/mcp_version.py -FORCE: +containers: container-base container-flow \ +container-bedrock container-vertexai \ +container-hf container-ocr \ +container-unstructured container-mcp -containers: FORCE - ${DOCKER} build -f containers/Containerfile.base \ - -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} . - ${DOCKER} build -f containers/Containerfile.flow \ - -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} . - ${DOCKER} build -f containers/Containerfile.bedrock \ - -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} . - ${DOCKER} build -f containers/Containerfile.vertexai \ - -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} . - ${DOCKER} build -f containers/Containerfile.hf \ - -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} . - ${DOCKER} build -f containers/Containerfile.ocr \ - -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} . - ${DOCKER} build -f containers/Containerfile.unstructured \ - -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} . - ${DOCKER} build -f containers/Containerfile.mcp \ - -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} . - -some-containers: - ${DOCKER} build -f containers/Containerfile.base \ - -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} . - ${DOCKER} build -f containers/Containerfile.flow \ - -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} . -# ${DOCKER} build -f containers/Containerfile.unstructured \ -# -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} . -# ${DOCKER} build -f containers/Containerfile.vertexai \ -# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} . -# ${DOCKER} build -f containers/Containerfile.mcp \ -# -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} . -# ${DOCKER} build -f containers/Containerfile.vertexai \ -# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} . -# ${DOCKER} build -f containers/Containerfile.bedrock \ -# -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} . - -basic-containers: update-package-versions - ${DOCKER} build -f containers/Containerfile.base \ - -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} . - ${DOCKER} build -f containers/Containerfile.flow \ - -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} . - -container.ocr: - ${DOCKER} build -f containers/Containerfile.ocr \ - -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} . +some-containers: container-base container-flow push: ${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION} @@ -109,54 +70,29 @@ push: ${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} # Individual container build targets -container-trustgraph-base: update-package-versions - ${DOCKER} build -f containers/Containerfile.base -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} . +container-%: update-package-versions + ${DOCKER} build \ + -f containers/Containerfile.${@:container-%=%} \ + -t ${CONTAINER_BASE}/trustgraph-${@:container-%=%}:${VERSION} . -container-trustgraph-flow: update-package-versions - ${DOCKER} build -f containers/Containerfile.flow -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} . +# Individual container build targets +manifest-%: update-package-versions + -@${DOCKER} manifest rm \ + ${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION} + ${DOCKER} build --platform linux/amd64,linux/arm64 \ + -f containers/Containerfile.${@:manifest-%=%} \ + --manifest \ + ${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION} . -container-trustgraph-bedrock: update-package-versions - ${DOCKER} build -f containers/Containerfile.bedrock -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} . +# Push a container +push-container-%: + ${DOCKER} push \ + ${CONTAINER_BASE}/trustgraph-${@:push-container-%=%}:${VERSION} -container-trustgraph-vertexai: update-package-versions - ${DOCKER} build -f containers/Containerfile.vertexai -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} . - -container-trustgraph-hf: update-package-versions - ${DOCKER} build -f containers/Containerfile.hf -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} . - -container-trustgraph-ocr: update-package-versions - ${DOCKER} build -f containers/Containerfile.ocr -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} . - -container-trustgraph-unstructured: update-package-versions - ${DOCKER} build -f containers/Containerfile.unstructured -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} . - -container-trustgraph-mcp: update-package-versions - ${DOCKER} build -f containers/Containerfile.mcp -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} . - -# Individual container push targets -push-trustgraph-base: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION} - -push-trustgraph-flow: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION} - -push-trustgraph-bedrock: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} - -push-trustgraph-vertexai: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} - -push-trustgraph-hf: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION} - -push-trustgraph-ocr: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} - -push-trustgraph-unstructured: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} - -push-trustgraph-mcp: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} +# Push a manifest +push-manifest-%: + ${DOCKER} manifest push \ + ${CONTAINER_BASE}/trustgraph-${@:push-manifest-%=%}:${VERSION} clean: rm -rf wheels/ @@ -164,52 +100,6 @@ clean: set-version: echo '"${VERSION}"' > templates/values/version.jsonnet -TEMPLATES=azure bedrock claude cohere mix llamafile mistral ollama openai vertexai \ - openai-neo4j storage - -DCS=$(foreach template,${TEMPLATES},${template:%=tg-launch-%.yaml}) - -MODELS=azure bedrock claude cohere llamafile mistral ollama openai vertexai -GRAPHS=cassandra neo4j falkordb memgraph - -# tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet -# jsonnet -Jtemplates \ -# -S ${@:tg-launch-%.yaml=templates/%.jsonnet} > $@ - -# VECTORDB=milvus -VECTORDB=qdrant - -JSONNET_FLAGS=-J templates -J . - -# Temporarily going back to how templates were built in 0.9 because this -# is going away in 0.11. - -update-templates: update-dcs - -JSON_TO_YAML=python -c 'import sys, yaml, json; j=json.loads(sys.stdin.read()); print(yaml.safe_dump(j))' - -update-dcs: set-version - for graph in ${GRAPHS}; do \ - cm=$${graph},pulsar,${VECTORDB},grafana; \ - input=templates/opts-to-docker-compose.jsonnet; \ - output=tg-storage-$${graph}.yaml; \ - echo $${graph} '->' $${output}; \ - jsonnet ${JSONNET_FLAGS} \ - --ext-str options=$${cm} $${input} | \ - ${JSON_TO_YAML} > $${output}; \ - done - for model in ${MODELS}; do \ - for graph in ${GRAPHS}; do \ - cm=$${graph},pulsar,${VECTORDB},embeddings-hf,graph-rag,grafana,trustgraph,$${model}; \ - input=templates/opts-to-docker-compose.jsonnet; \ - output=tg-launch-$${model}-$${graph}.yaml; \ - echo $${model} + $${graph} '->' $${output}; \ - jsonnet ${JSONNET_FLAGS} \ - --ext-str options=$${cm} $${input} | \ - ${JSON_TO_YAML} > $${output}; \ - done; \ - done - docker-hub-login: cat docker-token.txt | \ ${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io diff --git a/containers/Containerfile.hf b/containers/Containerfile.hf index 351300ae..250ad8e1 100644 --- a/containers/Containerfile.hf +++ b/containers/Containerfile.hf @@ -15,8 +15,10 @@ RUN dnf install -y python3.13 && \ pip3 install --no-cache-dir pulsar-client==3.7.0 && \ dnf clean all -RUN pip3 install torch==2.5.1+cpu \ - --index-url https://download.pytorch.org/whl/cpu +#RUN pip3 install torch==2.5.1+cpu \ +# --index-url https://download.pytorch.org/whl/cpu + +RUN pip3 install torch==2.5.1 RUN pip3 install --no-cache-dir \ langchain==0.3.25 langchain-core==0.3.60 langchain-huggingface==0.2.0 \