diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dc2fc89b..ec5fd31d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,6 +27,12 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # This allows the x86 runner to emulate ARM, needed to build ARM images + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - name: Get version id: version run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT @@ -80,8 +86,8 @@ jobs: run: make update-package-versions VERSION=${{ steps.version.outputs.VERSION }} - name: Build container - ${{ matrix.container }} - run: make container-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }} + run: make manifest-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }} - name: Push container - ${{ matrix.container }} - run: make push-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }} + run: make push-manifest-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }} diff --git a/Makefile b/Makefile index 197a6c63..593b7a9c 100644 --- a/Makefile +++ b/Makefile @@ -52,51 +52,12 @@ update-package-versions: echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py echo __version__ = \"${VERSION}\" > trustgraph-mcp/trustgraph/mcp_version.py -FORCE: +containers: container-base container-flow \ +container-bedrock container-vertexai \ +container-hf container-ocr \ +container-unstructured container-mcp -containers: FORCE - ${DOCKER} build -f containers/Containerfile.base \ - -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} . - ${DOCKER} build -f containers/Containerfile.flow \ - -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} . - ${DOCKER} build -f containers/Containerfile.bedrock \ - -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} . - ${DOCKER} build -f containers/Containerfile.vertexai \ - -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} . - ${DOCKER} build -f containers/Containerfile.hf \ - -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} . - ${DOCKER} build -f containers/Containerfile.ocr \ - -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} . - ${DOCKER} build -f containers/Containerfile.unstructured \ - -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} . - ${DOCKER} build -f containers/Containerfile.mcp \ - -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} . - -some-containers: - ${DOCKER} build -f containers/Containerfile.base \ - -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} . - ${DOCKER} build -f containers/Containerfile.flow \ - -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} . -# ${DOCKER} build -f containers/Containerfile.unstructured \ -# -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} . -# ${DOCKER} build -f containers/Containerfile.vertexai \ -# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} . -# ${DOCKER} build -f containers/Containerfile.mcp \ -# -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} . -# ${DOCKER} build -f containers/Containerfile.vertexai \ -# -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} . -# ${DOCKER} build -f containers/Containerfile.bedrock \ -# -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} . - -basic-containers: update-package-versions - ${DOCKER} build -f containers/Containerfile.base \ - -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} . - ${DOCKER} build -f containers/Containerfile.flow \ - -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} . - -container.ocr: - ${DOCKER} build -f containers/Containerfile.ocr \ - -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} . +some-containers: container-base container-flow push: ${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION} @@ -109,54 +70,29 @@ push: ${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} # Individual container build targets -container-trustgraph-base: update-package-versions - ${DOCKER} build -f containers/Containerfile.base -t ${CONTAINER_BASE}/trustgraph-base:${VERSION} . +container-%: update-package-versions + ${DOCKER} build \ + -f containers/Containerfile.${@:container-%=%} \ + -t ${CONTAINER_BASE}/trustgraph-${@:container-%=%}:${VERSION} . -container-trustgraph-flow: update-package-versions - ${DOCKER} build -f containers/Containerfile.flow -t ${CONTAINER_BASE}/trustgraph-flow:${VERSION} . +# Individual container build targets +manifest-%: update-package-versions + -@${DOCKER} manifest rm \ + ${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION} + ${DOCKER} build --platform linux/amd64,linux/arm64 \ + -f containers/Containerfile.${@:manifest-%=%} \ + --manifest \ + ${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION} . -container-trustgraph-bedrock: update-package-versions - ${DOCKER} build -f containers/Containerfile.bedrock -t ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} . +# Push a container +push-container-%: + ${DOCKER} push \ + ${CONTAINER_BASE}/trustgraph-${@:push-container-%=%}:${VERSION} -container-trustgraph-vertexai: update-package-versions - ${DOCKER} build -f containers/Containerfile.vertexai -t ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} . - -container-trustgraph-hf: update-package-versions - ${DOCKER} build -f containers/Containerfile.hf -t ${CONTAINER_BASE}/trustgraph-hf:${VERSION} . - -container-trustgraph-ocr: update-package-versions - ${DOCKER} build -f containers/Containerfile.ocr -t ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} . - -container-trustgraph-unstructured: update-package-versions - ${DOCKER} build -f containers/Containerfile.unstructured -t ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} . - -container-trustgraph-mcp: update-package-versions - ${DOCKER} build -f containers/Containerfile.mcp -t ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} . - -# Individual container push targets -push-trustgraph-base: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION} - -push-trustgraph-flow: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION} - -push-trustgraph-bedrock: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION} - -push-trustgraph-vertexai: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION} - -push-trustgraph-hf: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION} - -push-trustgraph-ocr: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-ocr:${VERSION} - -push-trustgraph-unstructured: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION} - -push-trustgraph-mcp: - ${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION} +# Push a manifest +push-manifest-%: + ${DOCKER} manifest push \ + ${CONTAINER_BASE}/trustgraph-${@:push-manifest-%=%}:${VERSION} clean: rm -rf wheels/ @@ -164,52 +100,6 @@ clean: set-version: echo '"${VERSION}"' > templates/values/version.jsonnet -TEMPLATES=azure bedrock claude cohere mix llamafile mistral ollama openai vertexai \ - openai-neo4j storage - -DCS=$(foreach template,${TEMPLATES},${template:%=tg-launch-%.yaml}) - -MODELS=azure bedrock claude cohere llamafile mistral ollama openai vertexai -GRAPHS=cassandra neo4j falkordb memgraph - -# tg-launch-%.yaml: templates/%.jsonnet templates/components/version.jsonnet -# jsonnet -Jtemplates \ -# -S ${@:tg-launch-%.yaml=templates/%.jsonnet} > $@ - -# VECTORDB=milvus -VECTORDB=qdrant - -JSONNET_FLAGS=-J templates -J . - -# Temporarily going back to how templates were built in 0.9 because this -# is going away in 0.11. - -update-templates: update-dcs - -JSON_TO_YAML=python -c 'import sys, yaml, json; j=json.loads(sys.stdin.read()); print(yaml.safe_dump(j))' - -update-dcs: set-version - for graph in ${GRAPHS}; do \ - cm=$${graph},pulsar,${VECTORDB},grafana; \ - input=templates/opts-to-docker-compose.jsonnet; \ - output=tg-storage-$${graph}.yaml; \ - echo $${graph} '->' $${output}; \ - jsonnet ${JSONNET_FLAGS} \ - --ext-str options=$${cm} $${input} | \ - ${JSON_TO_YAML} > $${output}; \ - done - for model in ${MODELS}; do \ - for graph in ${GRAPHS}; do \ - cm=$${graph},pulsar,${VECTORDB},embeddings-hf,graph-rag,grafana,trustgraph,$${model}; \ - input=templates/opts-to-docker-compose.jsonnet; \ - output=tg-launch-$${model}-$${graph}.yaml; \ - echo $${model} + $${graph} '->' $${output}; \ - jsonnet ${JSONNET_FLAGS} \ - --ext-str options=$${cm} $${input} | \ - ${JSON_TO_YAML} > $${output}; \ - done; \ - done - docker-hub-login: cat docker-token.txt | \ ${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io