mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-27 06:01:01 +02:00
bdfdbb3c1f
Add trustgraph-docling package providing a document decoder powered by IBM's Docling library. Supports PDF, DOCX, XLSX, PPTX, HTML, Markdown, and CSV with two chunking modes: - page mode (default): groups output by page for page-based formats, emits TextDocument messages for the existing downstream chunker - hybrid mode: uses Docling's HybridChunker for structure-aware semantic chunking, emits Chunk messages directly Tables are preserved as HTML markup. Images are stored in the librarian with provenance but excluded from the text pipeline. Full provenance chain is maintained for explainability tracing. Includes Containerfile, CI/CD integration (PR checks and release workflows), Makefile targets, and tech spec documentation. Added unit tests for the package.
142 lines
5.8 KiB
Makefile
142 lines
5.8 KiB
Makefile
|
|
# VERSION=$(shell git describe | sed 's/^v//')
|
|
|
|
VERSION=0.0.0
|
|
|
|
DOCKER=podman
|
|
|
|
all: containers
|
|
|
|
# Not used
|
|
wheels:
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph/
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph-base/
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph-flow/
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph-vertexai/
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph-bedrock/
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph-embeddings-hf/
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph-cli/
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph-ocr/
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph-unstructured/
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph-docling/
|
|
pip3 wheel --no-deps --wheel-dir dist trustgraph-mcp/
|
|
|
|
packages: update-package-versions
|
|
rm -rf dist/
|
|
cd trustgraph && python -m build --sdist --outdir ../dist/
|
|
cd trustgraph-base && python -m build --sdist --outdir ../dist/
|
|
cd trustgraph-flow && python -m build --sdist --outdir ../dist/
|
|
cd trustgraph-vertexai && python -m build --sdist --outdir ../dist/
|
|
cd trustgraph-bedrock && python -m build --sdist --outdir ../dist/
|
|
cd trustgraph-embeddings-hf && python -m build --sdist --outdir ../dist/
|
|
cd trustgraph-cli && python -m build --sdist --outdir ../dist/
|
|
cd trustgraph-ocr && python -m build --sdist --outdir ../dist/
|
|
cd trustgraph-unstructured && python -m build --sdist --outdir ../dist/
|
|
cd trustgraph-docling && python -m build --sdist --outdir ../dist/
|
|
cd trustgraph-mcp && python -m build --sdist --outdir ../dist/
|
|
|
|
pypi-upload:
|
|
twine upload dist/*-${VERSION}.*
|
|
|
|
CONTAINER_BASE=docker.io/trustgraph
|
|
|
|
update-package-versions:
|
|
mkdir -p trustgraph-cli/trustgraph
|
|
mkdir -p trustgraph/trustgraph
|
|
echo __version__ = \"${VERSION}\" > trustgraph-base/trustgraph/base_version.py
|
|
echo __version__ = \"${VERSION}\" > trustgraph-flow/trustgraph/flow_version.py
|
|
echo __version__ = \"${VERSION}\" > trustgraph-vertexai/trustgraph/vertexai_version.py
|
|
echo __version__ = \"${VERSION}\" > trustgraph-bedrock/trustgraph/bedrock_version.py
|
|
echo __version__ = \"${VERSION}\" > trustgraph-embeddings-hf/trustgraph/embeddings_hf_version.py
|
|
echo __version__ = \"${VERSION}\" > trustgraph-cli/trustgraph/cli_version.py
|
|
echo __version__ = \"${VERSION}\" > trustgraph-ocr/trustgraph/ocr_version.py
|
|
echo __version__ = \"${VERSION}\" > trustgraph-unstructured/trustgraph/unstructured_version.py
|
|
echo __version__ = \"${VERSION}\" > trustgraph-docling/trustgraph/docling_version.py
|
|
echo __version__ = \"${VERSION}\" > trustgraph/trustgraph/trustgraph_version.py
|
|
echo __version__ = \"${VERSION}\" > trustgraph-mcp/trustgraph/mcp_version.py
|
|
|
|
containers: container-base container-flow \
|
|
container-bedrock container-vertexai \
|
|
container-hf container-ocr \
|
|
container-unstructured container-docling container-mcp
|
|
|
|
some-containers: container-base container-flow
|
|
# container-unstructured
|
|
|
|
push:
|
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-base:${VERSION}
|
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-flow:${VERSION}
|
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-bedrock:${VERSION}
|
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-vertexai:${VERSION}
|
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-hf:${VERSION}
|
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-ocr:${VERSION}
|
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-unstructured:${VERSION}
|
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-docling:${VERSION}
|
|
${DOCKER} push ${CONTAINER_BASE}/trustgraph-mcp:${VERSION}
|
|
|
|
# Individual container build targets
|
|
container-%: update-package-versions
|
|
${DOCKER} build \
|
|
-f containers/Containerfile.${@:container-%=%} \
|
|
-t ${CONTAINER_BASE}/trustgraph-${@:container-%=%}:${VERSION} .
|
|
|
|
# Multi-arch: build both platforms sequentially into one manifest (local use)
|
|
manifest-%: update-package-versions
|
|
-@${DOCKER} manifest rm \
|
|
${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION}
|
|
${DOCKER} build --platform linux/amd64,linux/arm64 \
|
|
-f containers/Containerfile.${@:manifest-%=%} \
|
|
--manifest \
|
|
${CONTAINER_BASE}/trustgraph-${@:manifest-%=%}:${VERSION} .
|
|
|
|
# Multi-arch: build a single platform image (for parallel CI)
|
|
platform-%-amd64: update-package-versions
|
|
${DOCKER} build --platform linux/amd64 \
|
|
-f containers/Containerfile.${@:platform-%-amd64=%} \
|
|
-t ${CONTAINER_BASE}/trustgraph-${@:platform-%-amd64=%}:${VERSION}-amd64 .
|
|
|
|
platform-%-arm64: update-package-versions
|
|
${DOCKER} build --platform linux/arm64 \
|
|
-f containers/Containerfile.${@:platform-%-arm64=%} \
|
|
-t ${CONTAINER_BASE}/trustgraph-${@:platform-%-arm64=%}:${VERSION}-arm64 .
|
|
|
|
# Push a single platform image
|
|
push-platform-%-amd64:
|
|
${DOCKER} push \
|
|
${CONTAINER_BASE}/trustgraph-${@:push-platform-%-amd64=%}:${VERSION}-amd64
|
|
|
|
push-platform-%-arm64:
|
|
${DOCKER} push \
|
|
${CONTAINER_BASE}/trustgraph-${@:push-platform-%-arm64=%}:${VERSION}-arm64
|
|
|
|
# Combine per-platform images into a multi-arch manifest
|
|
combine-manifest-%:
|
|
-@${DOCKER} manifest rm \
|
|
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}
|
|
${DOCKER} manifest create \
|
|
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION} \
|
|
docker://${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}-amd64 \
|
|
docker://${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}-arm64
|
|
${DOCKER} manifest push \
|
|
${CONTAINER_BASE}/trustgraph-${@:combine-manifest-%=%}:${VERSION}
|
|
|
|
# Push a container
|
|
push-container-%:
|
|
${DOCKER} push \
|
|
${CONTAINER_BASE}/trustgraph-${@:push-container-%=%}:${VERSION}
|
|
|
|
# Push a manifest (from local multi-arch build)
|
|
push-manifest-%:
|
|
${DOCKER} manifest push \
|
|
${CONTAINER_BASE}/trustgraph-${@:push-manifest-%=%}:${VERSION}
|
|
|
|
clean:
|
|
rm -rf wheels/
|
|
|
|
set-version:
|
|
echo '"${VERSION}"' > templates/values/version.jsonnet
|
|
|
|
docker-hub-login:
|
|
cat docker-token.txt | \
|
|
${DOCKER} login -u trustgraph --password-stdin registry-1.docker.io
|
|
|