Use manifests to build for amd64 and arm64 (#798)

Remove Pytorch +cpu invocation, so it builds on ARM

.github/workflows/release.yaml

@@ -52,20 +52,25 @@ jobs:
     strategy:
       matrix:
         container:
-          - trustgraph-base
+          - base
-          - trustgraph-flow
+          - flow
-          - trustgraph-bedrock
+          - bedrock
-          - trustgraph-vertexai
+          - vertexai
-          - trustgraph-hf
+          - hf
-          - trustgraph-ocr
+          - ocr
-          - trustgraph-unstructured
+          - unstructured
-          - trustgraph-mcp
+          - mcp
 
     steps:
 
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Set up QEMU - for ARM emulation
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
       - name: Docker Hub token
         run: echo ${{ secrets.DOCKER_SECRET }} > docker-token.txt
 
@@ -80,8 +85,8 @@ jobs:
         run: make update-package-versions VERSION=${{ steps.version.outputs.VERSION }}
 
       - name: Build container - ${{ matrix.container }}
-        run: make container-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
+        run: make manifest-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
 
       - name: Push container - ${{ matrix.container }}
-        run: make push-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
+        run: make push-manifest-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
 

Makefile

@@ -52,51 +52,12 @@ update-package-versions:
 	echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
 	echo __version__ = \"${VERSION}\" > trustgraph-mcp/trustgraph/mcp_version.py
 
-FORCE:
+containers: container-base container-flow \
+container-bedrock container-vertexai \
+container-hf container-ocr \
+container-unstructured container-mcp
 
-containers: FORCE
+some-containers: container-base container-flow
-	${DOCKER} build -f containers/Containerfile.base \
-	    -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
-	${DOCKER} build -f containers/Containerfile.flow \
-	    -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
-	${DOCKER} build -f containers/Containerfile.bedrock \
-	    -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
-	${DOCKER} build -f containers/Containerfile.vertexai \
-	    -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
-	${DOCKER} build -f containers/Containerfile.hf \
-	    -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
-	${DOCKER} build -f containers/Containerfile.ocr \
-	    -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
-	${DOCKER} build -f containers/Containerfile.unstructured \
-	    -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
-	${DOCKER} build -f containers/Containerfile.mcp \
-	    -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
-
-some-containers:
-	${DOCKER} build -f containers/Containerfile.base \
-	    -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
-	${DOCKER} build -f containers/Containerfile.flow \
-	    -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
-#	${DOCKER} build -f containers/Containerfile.unstructured \
-#	    -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
-#	${DOCKER} build -f containers/Containerfile.vertexai \
-#	    -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
-#	${DOCKER} build -f containers/Containerfile.mcp \
-#	    -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
-#	${DOCKER} build -f containers/Containerfile.vertexai \
-#	    -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
-#	${DOCKER} build -f containers/Containerfile.bedrock \
-#	    -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
-
-basic-containers: update-package-versions
-	${DOCKER} build -f containers/Containerfile.base \
-	    -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
-	${DOCKER} build -f containers/Containerfile.flow \
-	    -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
-
-container.ocr:
-	${DOCKER} build -f containers/Containerfile.ocr \
-	    -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
 
 push:
 	${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
@@ -109,54 +70,29 @@ push:
 	${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
 
 # Individual container build targets
-container-trustgraph-base: update-package-versions
+container-%: update-package-versions
-	${DOCKER} build -f containers/Containerfile.base -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} .
+	${DOCKER} build \
+	    -f containers/Containerfile.${@:container-%=%} \
+	    -t ${CONTAINER_BASE}/trustgraph-${@:container-%=%}:${VERSION} .
 
-container-trustgraph-flow: update-package-versions
+# Individual container build targets
-	${DOCKER} build -f containers/Containerfile.flow -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} .
+manifest-%: update-package-versions
+	-@${DOCKER} manifest rm \
+	    ${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION}
+	${DOCKER} build --platform linux/amd64,linux/arm64 \
+	    -f containers/Containerfile.${@:manifest-%=%} \
+	    --manifest \
+	    ${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION} .
 
-container-trustgraph-bedrock: update-package-versions
+# Push a container
-	${DOCKER} build -f containers/Containerfile.bedrock -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} .
+push-container-%:
+	${DOCKER} push \
+	    ${CONTAINER_BASE}/trustgraph-${@:push-container-%=%}:${VERSION}
 
-container-trustgraph-vertexai: update-package-versions
+# Push a manifest
-	${DOCKER} build -f containers/Containerfile.vertexai -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} .
+push-manifest-%:
-
+	${DOCKER} manifest push \
-container-trustgraph-hf: update-package-versions
+	    ${CONTAINER_BASE}/trustgraph-${@:push-manifest-%=%}:${VERSION}
-	${DOCKER} build -f containers/Containerfile.hf -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} .
-
-container-trustgraph-ocr: update-package-versions
-	${DOCKER} build -f containers/Containerfile.ocr -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} .
-
-container-trustgraph-unstructured: update-package-versions
-	${DOCKER} build -f containers/Containerfile.unstructured -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} .
-
-container-trustgraph-mcp: update-package-versions
-	${DOCKER} build -f containers/Containerfile.mcp -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} .
-
-# Individual container push targets
-push-trustgraph-base:
-	${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
-
-push-trustgraph-flow:
-	${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION}
-
-push-trustgraph-bedrock:
-	${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION}
-
-push-trustgraph-vertexai:
-	${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION}
-
-push-trustgraph-hf:
-	${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION}
-
-push-trustgraph-ocr:
-	${DOCKER} push ${CONTAINER_BASE}/trustgraph-ocr:${VERSION}
-
-push-trustgraph-unstructured:
-	${DOCKER} push ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION}
-
-push-trustgraph-mcp:
-	${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
 
 clean:
 	rm -rf wheels/
@@ -164,52 +100,6 @@ clean:
 set-version:
 	echo '"${VERSION}"' > templates/values/version.jsonnet
 
-TEMPLATES=azure bedrock claude cohere mix llamafile mistral ollama openai vertexai \
-    openai-neo4j storage
-
-DCS=$(foreach template,${TEMPLATES},${template:%=tg-launch-%.yaml})
-
-MODELS=azure bedrock claude cohere llamafile mistral ollama openai vertexai
-GRAPHS=cassandra neo4j falkordb memgraph
-
-# tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet
-# 	jsonnet -Jtemplates \
-# 	    -S ${@:tg-launch-%.yaml=templates/%.jsonnet} > $@
-
-# VECTORDB=milvus
-VECTORDB=qdrant
-
-JSONNET_FLAGS=-J templates -J .
-
-# Temporarily going back to how templates were built in 0.9 because this
-# is going away in 0.11.
-
-update-templates: update-dcs
-
-JSON_TO_YAML=python -c 'import sys, yaml, json; j=json.loads(sys.stdin.read()); print(yaml.safe_dump(j))'
-
-update-dcs: set-version
-	for graph in ${GRAPHS}; do \
-	    cm=$${graph},pulsar,${VECTORDB},grafana; \
-	    input=templates/opts-to-docker-compose.jsonnet; \
-	    output=tg-storage-$${graph}.yaml; \
-	    echo $${graph} '->' $${output}; \
-	    jsonnet ${JSONNET_FLAGS} \
-	         --ext-str options=$${cm} $${input} | \
-	         ${JSON_TO_YAML} > $${output}; \
-	done
-	for model in ${MODELS}; do \
-	  for graph in ${GRAPHS}; do \
-	    cm=$${graph},pulsar,${VECTORDB},embeddings-hf,graph-rag,grafana,trustgraph,$${model}; \
-	    input=templates/opts-to-docker-compose.jsonnet; \
-	    output=tg-launch-$${model}-$${graph}.yaml; \
-	    echo $${model} + $${graph} '->' $${output}; \
-	    jsonnet ${JSONNET_FLAGS} \
-	         --ext-str options=$${cm} $${input} | \
-                 ${JSON_TO_YAML} > $${output}; \
-	  done; \
-	done
-
 docker-hub-login:
 	cat docker-token.txt | \
 	    ${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io

containers/Containerfile.hf

@@ -15,8 +15,10 @@ RUN dnf install -y python3.13 && \
   pip3 install --no-cache-dir pulsar-client==3.7.0 && \
   dnf clean all
 
-RUN pip3 install torch==2.5.1+cpu \
+#RUN pip3 install torch==2.5.1+cpu \
-    --index-url https://download.pytorch.org/whl/cpu
+#    --index-url https://download.pytorch.org/whl/cpu
+
+RUN pip3 install torch==2.5.1
 
 RUN pip3 install --no-cache-dir \
     langchain==0.3.25 langchain-core==0.3.60 langchain-huggingface==0.2.0 \