mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-27 06:01:01 +02:00
bdfdbb3c1f
Add trustgraph-docling package providing a document decoder powered by IBM's Docling library. Supports PDF, DOCX, XLSX, PPTX, HTML, Markdown, and CSV with two chunking modes: - page mode (default): groups output by page for page-based formats, emits TextDocument messages for the existing downstream chunker - hybrid mode: uses Docling's HybridChunker for structure-aware semantic chunking, emits Chunk messages directly Tables are preserved as HTML markup. Images are stored in the librarian with provenance but excluded from the text pipeline. Full provenance chain is maintained for explainability tracing. Includes Containerfile, CI/CD integration (PR checks and release workflows), Makefile targets, and tech spec documentation. Added unit tests for the package.
141 lines
3.3 KiB
YAML
141 lines
3.3 KiB
YAML
|
|
name: Build
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
push:
|
|
tags:
|
|
- v*
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
|
|
python-packages:
|
|
|
|
name: Release Python packages
|
|
runs-on: ubuntu-24.04
|
|
permissions:
|
|
contents: write
|
|
id-token: write
|
|
environment:
|
|
name: release
|
|
|
|
steps:
|
|
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Get version
|
|
id: version
|
|
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
|
|
|
- name: Install dependencies
|
|
run: pip install build wheel
|
|
|
|
- name: Build packages
|
|
run: make packages VERSION=${{ steps.version.outputs.VERSION }}
|
|
|
|
- name: Publish release distributions to PyPI
|
|
uses: pypa/gh-action-pypi-publish@release/v1
|
|
|
|
build-platform-image:
|
|
|
|
name: Build ${{ matrix.container }} (${{ matrix.platform }})
|
|
permissions:
|
|
contents: write
|
|
id-token: write
|
|
environment:
|
|
name: release
|
|
strategy:
|
|
matrix:
|
|
container:
|
|
- base
|
|
- flow
|
|
- bedrock
|
|
- vertexai
|
|
- hf
|
|
- ocr
|
|
- unstructured
|
|
- docling
|
|
- mcp
|
|
platform:
|
|
- amd64
|
|
- arm64
|
|
include:
|
|
- platform: amd64
|
|
runner: ubuntu-24.04
|
|
- platform: arm64
|
|
runner: ubuntu-24.04-arm
|
|
|
|
runs-on: ${{ matrix.runner }}
|
|
|
|
steps:
|
|
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: "Free up some disk space"
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
|
|
sudo rm -rf /opt/hostedtoolcache/CodeQL
|
|
podman image prune --all --force
|
|
podman builder prune -a -f
|
|
|
|
- name: Docker Hub token
|
|
run: echo ${{ secrets.DOCKER_SECRET }} > docker-token.txt
|
|
|
|
- name: Authenticate with Docker hub
|
|
run: make docker-hub-login
|
|
|
|
- name: Get version
|
|
id: version
|
|
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
|
|
|
- name: Build container
|
|
run: make platform-${{ matrix.container }}-${{ matrix.platform }} VERSION=${{ steps.version.outputs.VERSION }}
|
|
|
|
- name: Push container
|
|
run: make push-platform-${{ matrix.container }}-${{ matrix.platform }} VERSION=${{ steps.version.outputs.VERSION }}
|
|
|
|
combine-manifests:
|
|
|
|
name: Combine manifest ${{ matrix.container }}
|
|
runs-on: ubuntu-24.04
|
|
needs: build-platform-image
|
|
permissions:
|
|
contents: write
|
|
id-token: write
|
|
environment:
|
|
name: release
|
|
strategy:
|
|
matrix:
|
|
container:
|
|
- base
|
|
- flow
|
|
- bedrock
|
|
- vertexai
|
|
- hf
|
|
- ocr
|
|
- unstructured
|
|
- docling
|
|
- mcp
|
|
|
|
steps:
|
|
|
|
- name: Checkout
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Docker Hub token
|
|
run: echo ${{ secrets.DOCKER_SECRET }} > docker-token.txt
|
|
|
|
- name: Authenticate with Docker hub
|
|
run: make docker-hub-login
|
|
|
|
- name: Get version
|
|
id: version
|
|
run: echo VERSION=$(git describe --exact-match --tags | sed 's/^v//') >> $GITHUB_OUTPUT
|
|
|
|
- name: Combine and push manifest
|
|
run: make combine-manifest-${{ matrix.container }} VERSION=${{ steps.version.outputs.VERSION }}
|