mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-27 14:11:02 +02:00
bdfdbb3c1f
Add trustgraph-docling package providing a document decoder powered by IBM's Docling library. Supports PDF, DOCX, XLSX, PPTX, HTML, Markdown, and CSV with two chunking modes: - page mode (default): groups output by page for page-based formats, emits TextDocument messages for the existing downstream chunker - hybrid mode: uses Docling's HybridChunker for structure-aware semantic chunking, emits Chunk messages directly Tables are preserved as HTML markup. Images are stored in the librarian with provenance but excluded from the text pipeline. Full provenance chain is maintained for explainability tracing. Includes Containerfile, CI/CD integration (PR checks and release workflows), Makefile targets, and tech spec documentation. Added unit tests for the package.
57 lines
1.9 KiB
Text
57 lines
1.9 KiB
Text
|
|
# ----------------------------------------------------------------------------
|
|
# Base container with system dependencies
|
|
# ----------------------------------------------------------------------------
|
|
|
|
FROM docker.io/fedora:42 AS base
|
|
|
|
ENV PIP_BREAK_SYSTEM_PACKAGES=1
|
|
|
|
# This is cached, same as other containers
|
|
RUN dnf install -y python3.11 && \
|
|
alternatives --install /usr/bin/python python /usr/bin/python3.11 1 && \
|
|
python -m ensurepip --upgrade && \
|
|
pip3 install --no-cache-dir --upgrade 'pip>=26.0' 'setuptools>=78.1.1' && \
|
|
pip3 install --no-cache-dir build wheel aiohttp && \
|
|
pip3 install --no-cache-dir pulsar-client==3.11.0 && \
|
|
dnf clean all
|
|
|
|
# Installs local to me
|
|
RUN pip install --no-cache-dir \
|
|
torch torchvision --index-url https://download.pytorch.org/whl/cpu
|
|
|
|
RUN dnf install -y libxcb mesa-libGL && \
|
|
dnf clean all
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Build a container which contains the built Python packages. The build
|
|
# creates a bunch of left-over cruft, a separate phase means this is only
|
|
# needed to support package build
|
|
# ----------------------------------------------------------------------------
|
|
|
|
FROM base AS build
|
|
|
|
COPY trustgraph-base/ /root/build/trustgraph-base/
|
|
COPY trustgraph-docling/ /root/build/trustgraph-docling/
|
|
|
|
WORKDIR /root/build/
|
|
|
|
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-base/
|
|
RUN pip3 wheel -w /root/wheels/ --no-deps ./trustgraph-docling/
|
|
|
|
RUN ls /root/wheels
|
|
|
|
# ----------------------------------------------------------------------------
|
|
# Finally, the target container. Start with base and add the package.
|
|
# ----------------------------------------------------------------------------
|
|
|
|
FROM base
|
|
|
|
COPY --from=build /root/wheels /root/wheels
|
|
|
|
RUN \
|
|
pip3 install --no-cache-dir /root/wheels/trustgraph_base-* && \
|
|
pip3 install --no-cache-dir /root/wheels/trustgraph_docling-* && \
|
|
rm -rf /root/wheels
|
|
|
|
WORKDIR /
|