diff --git a/ai-context/trustgraph-templates/.github/workflows/cla.yml b/ai-context/trustgraph-templates/.github/workflows/cla.yml new file mode 100644 index 00000000..73f582cf --- /dev/null +++ b/ai-context/trustgraph-templates/.github/workflows/cla.yml @@ -0,0 +1,25 @@ +name: CLA Assistant + +on: + issue_comment: + types: [created] + pull_request_target: + types: [opened, synchronize, reopened] + +permissions: + actions: write + contents: write + pull-requests: write + statuses: write + +jobs: + CLAssistant: + runs-on: ubuntu-latest + steps: + - name: CLA Assistant + uses: trustgraph-ai/contributor-license-agreement/action@main + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.CLA_ASSISTANT_PAT }} + with: + allowlist: 'dependabot,dependabot[bot],github-actions,github-actions[bot]' diff --git a/ai-context/trustgraph-templates/.github/workflows/deploy-prod.yaml b/ai-context/trustgraph-templates/.github/workflows/deploy-prod.yaml new file mode 100644 index 00000000..cb902a52 --- /dev/null +++ b/ai-context/trustgraph-templates/.github/workflows/deploy-prod.yaml @@ -0,0 +1,76 @@ + +name: Deploy to prod + +on: + workflow_dispatch: + push: + # Deploys on master branch + branches: + - master + +permissions: + contents: read + id-token: 'write' + packages: read + +jobs: + + deploy: + + name: Deploy to prod + runs-on: ubuntu-latest + + steps: + + - name: Checkout + uses: actions/checkout@v3 + + - name: Get version + id: version + run: echo VERSION=sha-$(git rev-parse --short HEAD) >> $GITHUB_OUTPUT + + # Python package version MUST be a semantic version, but also doesn't + # matter, so just setting to 0.0.0. + # The container version MUST change on every push to get Cloud Run + # to re-deploy, so is based on git hash. + - name: Build container + run: make PACKAGE_VERSION=0.0.0 VERSION=${{ steps.version.outputs.VERSION }} + + - name: Log in to the container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: auth + name: Authenticate with Google Cloud + uses: google-github-actions/auth@v2 + with: + token_format: access_token + workload_identity_provider: projects/351149249312/locations/global/workloadIdentityPools/deploy/providers/github + service_account: deploy@trustgraph-ai.iam.gserviceaccount.com + access_token_lifetime: 900s + create_credentials_file: true + + - name: Login to Artifact Registry + uses: redhat-actions/podman-login@v1 + with: + registry: europe-west1-docker.pkg.dev + username: oauth2accesstoken + password: ${{ steps.auth.outputs.access_token }} + + - name: Install Pulumi + run: cd pulumi && npm install + + - name: Applying infrastructure πŸš€πŸ™ + uses: pulumi/actions@v3 + with: + command: up + stack-name: prod + work-dir: pulumi + cloud-url: gs://trustgraph-ai-deploy/config-svc + env: + PULUMI_CONFIG_PASSPHRASE: "" + IMAGE_VERSION: ${{ steps.version.outputs.VERSION }} + diff --git a/ai-context/trustgraph-templates/.github/workflows/pull-request.yaml b/ai-context/trustgraph-templates/.github/workflows/pull-request.yaml new file mode 100644 index 00000000..dbe64b15 --- /dev/null +++ b/ai-context/trustgraph-templates/.github/workflows/pull-request.yaml @@ -0,0 +1,31 @@ + +name: Test pull request + +on: + pull_request: + +permissions: + contents: read + +jobs: + + container-push: + + name: Run tests + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup environment + run: | + python3 -m venv env + . env/bin/activate + pip install -e .[dev] + + - name: Run pytest tests + run: | + . env/bin/activate + pytest -v --tb=short + diff --git a/ai-context/trustgraph-templates/.github/workflows/undeploy-prod.yaml b/ai-context/trustgraph-templates/.github/workflows/undeploy-prod.yaml new file mode 100644 index 00000000..d5cd2628 --- /dev/null +++ b/ai-context/trustgraph-templates/.github/workflows/undeploy-prod.yaml @@ -0,0 +1,45 @@ + +name: Undeploy to prod + +on: + workflow_dispatch: + +permissions: + contents: read + id-token: 'write' + +jobs: + + deploy: + + name: Undeploy to prod + runs-on: ubuntu-latest + + steps: + + - name: Checkout + uses: actions/checkout@v3 + + - id: auth + name: Authenticate with Google Cloud + uses: google-github-actions/auth@v2 + with: + token_format: access_token + workload_identity_provider: projects/514167726704/locations/global/workloadIdentityPools/deploy/providers/deploy + service_account: deploy@kalntera-demo.iam.gserviceaccount.com + access_token_lifetime: 900s + create_credentials_file: true + + - name: Install Pulumi + run: cd pulumi && npm install + + - name: Destroy infrastructure ☠πŸ”₯ + uses: pulumi/actions@v3 + with: + command: destroy + stack-name: prod + work-dir: pulumi + cloud-url: gs://trustgraph-deploy/config-ui + env: + PULUMI_CONFIG_PASSPHRASE: "" + diff --git a/ai-context/trustgraph-templates/.gitignore b/ai-context/trustgraph-templates/.gitignore new file mode 100644 index 00000000..6a5e4738 --- /dev/null +++ b/ai-context/trustgraph-templates/.gitignore @@ -0,0 +1,22 @@ +*~ +__pycache__ +*.pyc +*.pyo +*.egg-info/ +build/ +dist/ +*.egg +.pytest_cache/ +.coverage +htmlcov/ +.eggs/ +*.so +*.dylib +.Python +env/ +venv/ +ENV/ +*.log +node_modules/ +trustgraph_configurator/version.py +INSTALLATION.md diff --git a/ai-context/trustgraph-templates/Containerfile b/ai-context/trustgraph-templates/Containerfile new file mode 100644 index 00000000..4eba036e --- /dev/null +++ b/ai-context/trustgraph-templates/Containerfile @@ -0,0 +1,38 @@ +# --- STAGE 1: Build --- +FROM python:3.14-slim AS build + +# Install build tools (Debian uses apt) +RUN apt-get update && apt-get install -y \ + build-essential \ + golang \ + git \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /root/wheels /root/build + +# Build wheels +RUN pip wheel -w /root/wheels --no-deps gojsonnet + +COPY trustgraph_configurator/ /root/build/trustgraph_configurator/ +COPY pyproject.toml /root/build/pyproject.toml +COPY README.md /root/build/README.md + +RUN (cd /root/build && pip wheel -w /root/wheels --no-deps .) + +# --- STAGE 2: Runtime --- +FROM python:3.14-slim + +# No need to install libstdc++ manually, it's included in python-slim +RUN apt-get update && apt-get install -y \ + && rm -rf /var/lib/apt/lists/* + +# aiohttp and others will be pulled from PyPI or handled via wheels +COPY --from=build /root/wheels /root/wheels + +# Install your wheels plus regular dependencies +RUN pip install --no-cache-dir /root/wheels/* aiohttp pyyaml tabulate && \ + rm -rf /root/wheels + +CMD ["tg-config-svc"] +EXPOSE 8080 + diff --git a/ai-context/trustgraph-templates/DIALOG-FLOW.md b/ai-context/trustgraph-templates/DIALOG-FLOW.md new file mode 100644 index 00000000..e70ca46b --- /dev/null +++ b/ai-context/trustgraph-templates/DIALOG-FLOW.md @@ -0,0 +1,168 @@ +# TrustGraph Configuration Dialog Flow + +## Overview + +A configuration wizard system that guides users through TrustGraph deployment setup. Outputs deployment configuration and contextual documentation. + +Supports two interfaces: +- **Web UI**: Step-by-step wizard with visual cards +- **CLI**: Interactive terminal prompts + +## Architecture + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ trustgraph-flow β”‚ State machine defining wizard steps +β”‚ .yaml β”‚ and transitions +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Flow Engine β”‚ Executes state machine, collects user input, +β”‚ (Web UI / CLI) β”‚ manages state and backtracking +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Wizard State β”‚ Simple key/value object + β”‚ (JSON) β”‚ e.g., { platform: "gke", model_deployment: "ollama", ... } + β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”Œβ”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β” + β–Ό β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Output β”‚ β”‚Documentationβ”‚ +β”‚Transformβ”‚ β”‚ Assembler β”‚ +β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β–Ό β–Ό +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚Componentβ”‚ β”‚ README.md β”‚ +β”‚ Array β”‚ β”‚ / Checklist β”‚ +β”‚ (JSON) β”‚ β”‚ (JSON) β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +## Key Design Decisions + +| Decision | Choice | Rationale | +|----------|--------|-----------| +| Flow structure | State machine | Explicit transitions, supports conditional branching | +| Expression language | JSONata | Single language for conditions and transforms | +| Documentation storage | Manifest + markdown fragments | Clean separation, easy to maintain | +| Platform variants | `in ['docker-compose', 'podman-compose']` | Explicit grouping, no hidden logic | +| Template variables | `{{var}}` syntax | Simple, familiar | + +## File Structure + +``` +config-dialog-flow/ +β”œβ”€β”€ dialog-flow-schema.json # Schema: flow definitions +β”œβ”€β”€ docs-manifest-schema.json # Schema: documentation manifests +β”œβ”€β”€ trustgraph-flow.yaml # Flow: wizard state machine +β”œβ”€β”€ trustgraph-output.jsonata # Transform: state β†’ components +β”œβ”€β”€ trustgraph-docs.yaml # Manifest: state β†’ documentation +└── docs/ # Fragments: markdown content + β”œβ”€β”€ platform/ + β”œβ”€β”€ model/ + β”œβ”€β”€ storage/ + β”œβ”€β”€ gateway/ + β”œβ”€β”€ deploy/ + └── features/ +``` + +## Data Flow + +1. **Flow engine** loads `trustgraph-flow.yaml` +2. User progresses through steps, engine collects state +3. On completion: + - **Output transform** evaluates `trustgraph-output.jsonata` against state β†’ component array + - **Doc assembler** evaluates `trustgraph-docs.yaml` conditions against state β†’ filtered instructions β†’ loads markdown fragments β†’ outputs README or checklist JSON + +## Implementation Phases + +### Phase 1: Core Flow Engine + +- [ ] Parse YAML flow definition +- [ ] Implement state machine executor +- [ ] Handle step rendering (title, description, input) +- [ ] Evaluate JSONata transition conditions +- [ ] Manage wizard state (set values, navigate) +- [ ] Implement backtracking (previous step, state rollback) + +### Phase 2: Input Types + +- [ ] Select (single choice from options) +- [ ] Toggle (boolean) +- [ ] Number (with min/max/step) +- [ ] Text (free input) +- [ ] Skip single-option steps (UX decision) + +### Phase 3: Output Generation + +- [ ] Load JSONata transform +- [ ] Evaluate against wizard state +- [ ] Produce component array JSON +- [ ] Package as downloadable ZIP (existing functionality) + +### Phase 4: Documentation Assembly + +- [ ] Parse documentation manifest +- [ ] Evaluate `when` conditions (JSONata) +- [ ] Filter to matching instructions +- [ ] Load markdown fragments +- [ ] Substitute `{{var}}` placeholders +- [ ] Output as README.md (CLI) or checklist JSON (Web) + +### Phase 5: CLI Interface + +- [ ] Terminal prompt for each step +- [ ] Numbered option selection +- [ ] Progress indicator (`[3/12]`) +- [ ] Back command +- [ ] Review summary before generate + +### Phase 6: Web Interface + +- [ ] Step-by-step card UI +- [ ] Progress bar +- [ ] Option cards with icons/descriptions +- [ ] Back navigation +- [ ] Review screen with edit capability +- [ ] Generate button + +## Expression Language + +All conditions use JSONata: + +``` +platform = 'docker-compose' +platform in ['gke', 'eks', 'aks', 'minikube'] +model_deployment = 'ollama' and platform in ['docker-compose', 'podman-compose'] +version < '1.6.0' +ocr.enabled = true +``` + +## State Shape + +``` +{ + "version": "1.8.18", + "platform": "gke", + "k8s": { "namespace": "trustgraph" }, + "graph_store": "cassandra", + "vector_db": "qdrant", + "object_store": "cassandra", + "model_deployment": "ollama", + "max_output_tokens": 2048, + "ocr": { "enabled": true, "engine": "tesseract" }, + "embeddings": { "enabled": false } +} +``` + +## Out of Scope (Future) + +- Dual model mode (separate main/RAG models) +- Advanced settings (concurrency, chunking, memory profiles) +- Configuration import/export +- Validation beyond type checking diff --git a/ai-context/trustgraph-templates/Makefile b/ai-context/trustgraph-templates/Makefile new file mode 100644 index 00000000..ef0cdfec --- /dev/null +++ b/ai-context/trustgraph-templates/Makefile @@ -0,0 +1,22 @@ + +PACKAGE_VERSION=0.0.0 +VERSION=0.0.0 + +all: container + +package: update-package-versions + python3 -m build --sdist --outdir pkgs + +update-package-versions: + echo __version__ = \"${PACKAGE_VERSION}\" > trustgraph_configurator/version.py + +CONTAINER=localhost/config-svc +DOCKER=podman + +container: + ${DOCKER} build -f Containerfile -t ${CONTAINER}:${VERSION} \ + --format docker + +# On port 8081 +run-container: + ${DOCKER} run -i -t -p 8081:8080 ${CONTAINER}:${VERSION} diff --git a/ai-context/trustgraph-templates/README.md b/ai-context/trustgraph-templates/README.md new file mode 100644 index 00000000..a83bbbbc --- /dev/null +++ b/ai-context/trustgraph-templates/README.md @@ -0,0 +1,311 @@ +# TrustGraph Configuration Templates + +TrustGraph configurator is a Python-based tool that generates deployment configurations for TrustGraph AI systems. It supports multiple deployment platforms and provides templated configurations with versioning support. + +## Overview + +The configurator uses Jsonnet templates to generate deployment configurations for various platforms including Docker Compose, Podman, and multiple Kubernetes environments. It packages the generated configurations into ZIP files containing all necessary deployment resources. + +## Configuration Process + +The TrustGraph configuration system uses a multi-stage pipeline to generate deployment packages: + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Dialog Flow β”‚ β”‚ JSONata β”‚ β”‚ Configuration β”‚ β”‚ Deployment β”‚ +β”‚ Configuration │────▢│ Transform │────▢│ Service │────▢│ Package β”‚ +β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ β”‚ +β”‚ (state object) β”‚ β”‚ (config object) β”‚ β”‚ (templates) β”‚ β”‚ (ZIP file) β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + └──────────────▢│ Documentation │────▢│ Installation β”‚ + β”‚ Flow β”‚ β”‚ Guide β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### 1. Dialog Flow Configuration + +The dialog flow file (`trustgraph-flow.yaml`) describes configuration steps in a technology-neutral way. A UI wizard walks users through these steps, collecting choices about platform, model provider, storage backends, and features. The output is a **state object** - a simple key/value map representing all user selections. + +### 2. JSONata Transform + +The JSONata transform file (`trustgraph-output.jsonata`) converts the state object into a **configuration object**. This object understands how to invoke TrustGraph templates and contains the structured parameters needed by the template system. + +### 3. Configuration Service + +The configuration service receives the configuration object, invokes the appropriate Jsonnet templates, and runs the package builder. The output is a **deployment package** - a ZIP file containing all deployment resources (docker-compose.yaml or Kubernetes manifests, plus supporting files). + +### 4. Documentation Flow + +The original state object can also be used with the documentation manifest (`trustgraph-docs.yaml`) to generate a customised **installation guide** based on the user's specific configuration choices. + +## Installation + +The configurator is distributed as a Python package. To use it: + +```bash +export PYTHONPATH=. +# or install the package +pip install -e . +``` + +## Usage + +### List Available Configurations + +To see all available templates and platforms: + +```bash +tg-show-config-params +``` + +This will display: +- Available platforms (docker-compose, podman-compose, various Kubernetes options) +- Available templates with versions and stability status +- Latest version and latest stable version + +### Generate Configuration + +To generate a configuration package: + +```bash +scripts/tg-build-deployment --template --version \ + --input config.json --output output.zip --platform +``` + +Example: +```bash +scripts/tg-build-deployment --template 1.1 --version 1.1.9 \ + --input config.json --output deployment.zip --platform docker-compose +``` + +#### Output to stdout + +To output only the TrustGraph configuration: +```bash +scripts/tg-build-deployment --template 1.1 --latest-stable \ + --input config.json -O > trustgraph-config.json +``` + +To output only the platform resources (docker-compose.yaml or resources.yaml): +```bash +# For Docker Compose +scripts/tg-build-deployment --template 1.1 --latest-stable \ + --input config.json --platform docker-compose -R > docker-compose.yaml + +# For Kubernetes +scripts/tg-build-deployment --template 1.1 --latest-stable \ + --input config.json --platform gcp-k8s -R > resources.yaml +``` + +### Configuration Service API + +You can also run the configurator as a REST API service: + +```bash +scripts/tg-config-svc +``` + +This starts a web service on port 8080 that provides: +- REST API endpoints for configuration generation +- Programmatic access to version information +- Web-based configuration generation + +The service provides the same functionality as the command-line tool but through HTTP endpoints (see API Service section below for details). + +### Command Line Options + +- `-i, --input`: Input configuration file (default: config.json) +- `-o, --output`: Output ZIP file (default: output.zip) +- `-t, --template`: Template name (e.g., "1.1", "1.0", "0.23") +- `-v, --version`: Specific version to use +- `-p, --platform`: Target platform (default: docker-compose) +- `--latest`: Use the latest available version +- `--latest-stable`: Use the latest stable version +- `-O, --output-tg-config`: Output only TrustGraph configuration to stdout (no ZIP file) +- `-R, --output-resources`: Output only platform resources (docker-compose.yaml or resources.yaml) to stdout (no ZIP file) + +### Available Platforms + +- `docker-compose`: Local Docker deployment using docker-compose +- `podman-compose`: Local Podman deployment using podman-compose +- `minikube-k8s`: Minikube Kubernetes cluster +- `gcp-k8s`: Google Cloud Kubernetes (GKE) +- `aks-k8s`: Azure Kubernetes Service (AKS) +- `eks-k8s`: AWS Elastic Kubernetes Service (EKS) +- `scw-k8s`: Scaleway Kubernetes + +## Python Architecture + +### Module Structure + +The `trustgraph_configurator` package consists of several key modules: + +#### Core Modules + +1. **generator.py** (`Generator` class) + - Processes Jsonnet templates using the `_jsonnet` library + - Evaluates configuration snippets with custom import callbacks + - Returns processed JSON configurations + +2. **packager.py** (`Packager` class) + - Main orchestrator for configuration generation + - Handles template and resource file loading + - Generates platform-specific deployment packages + - Creates ZIP archives with all necessary files + - Supports both Docker Compose and Kubernetes outputs + +3. **index.py** (`Index` class) + - Manages template and platform metadata + - Reads from `templates/index.json` + - Provides version sorting and comparison + - Offers methods to get latest/stable versions + +4. **api.py** (`Api` class) + - REST API service for configuration generation + - Endpoints for version information and generation + - Validates input JSON before processing + - Returns generated configurations as binary data + +5. **service.py** + - Simple wrapper to run the API service + - Configures logging and starts the web server on port 8080 + +6. **run.py** + - Command-line interface implementation + - Argument parsing and validation + - Reads input configuration and writes output ZIP + +7. **list.py** + - Command-line tool to list available configurations + - Displays platforms, templates, and versions in tabular format + +### How Components Interact + +``` +User Input (config.json) + ↓ +run.py (CLI) or api.py (REST) + ↓ +Packager (orchestrator) + β”œβ”€β†’ Index (metadata/versions) + β”œβ”€β†’ Generator (Jsonnet processing) + └─→ Resource files (templates/) + ↓ + Platform-specific generation + (Docker Compose or Kubernetes) + ↓ + ZIP archive (output.zip) +``` + +### Key Design Patterns + +1. **Template Resolution**: The `Packager.fetch()` method implements a sophisticated file resolution system: + - Special handling for `trustgraph/config.json` and `version.jsonnet` + - Fallback search paths for templates and resources + - Version-specific template directories + +2. **Platform Abstraction**: Different platforms are handled through: + - Platform-specific Jsonnet templates (e.g., `config-to-docker-compose.jsonnet`) + - Conditional logic in `Packager.generate()` + - Unified output format (ZIP archives) + +3. **Version Management**: The system supports: + - Multiple template versions with different features + - Stability levels (alpha, beta, stable) + - Automatic version selection (latest/latest-stable) + +### Configuration Flow + +1. User provides a JSON configuration file +2. Packager validates and loads the appropriate template version +3. Generator processes Jsonnet templates with the configuration +4. Platform-specific resources are added (Grafana dashboards, Prometheus config) +5. Everything is packaged into a ZIP file for deployment + +### API Service + +The REST API service (`tg-config-svc`) provides programmatic access to the configurator functionality. Start the service with: + +```bash +scripts/tg-config-svc +``` + +The service runs on port 8080 and provides the following endpoints: + +``` +POST /api/generate/{platform}/{template} # Generate configuration +GET /api/latest # Get latest version info +GET /api/latest-stable # Get latest stable version info +GET /api/versions # List all available versions +``` + +#### Dialog Flow Resources + +These endpoints serve the dialog flow resources described in the Configuration Process section: + +``` +GET /api/dialog-flow # Dialog flow state machine (YAML) +GET /api/config-prepare # JSONata transform for config preparation +GET /api/docs-manifest # Documentation manifest (YAML) +GET /api/docs/{path} # Documentation markdown fragments +``` + +Example usage: +```bash +# Generate configuration via API +curl -X POST http://localhost:8080/api/generate/docker-compose/1.1 \ + -H "Content-Type: application/json" \ + -d @config.json \ + --output deployment.zip + +# Fetch dialog flow configuration +curl http://localhost:8080/api/dialog-flow + +# Fetch a documentation fragment +curl http://localhost:8080/api/docs/platform/docker-compose.md +``` + +## Output Structure + +### Docker Compose Output + +The generated ZIP file contains: +``` +docker-compose.yaml # Main deployment file +trustgraph/config.json # TrustGraph configuration +grafana/ # Grafana dashboards and provisioning +prometheus/ # Prometheus configuration +``` + +### Kubernetes Output + +The generated ZIP file contains: +``` +resources.yaml # All Kubernetes resources in a single file +``` + +## Development + +To extend or modify the configurator: + +1. Templates are in `trustgraph_configurator/templates//` +2. Add new platforms by creating appropriate Jsonnet templates +3. Update `templates/index.json` for new versions +4. Resources (dashboards, configs) go in `trustgraph_configurator/resources//` +5. Dialog flow resources are in `trustgraph_configurator/resources/dialog/`: + - `trustgraph-flow.yaml` - Dialog flow state machine + - `trustgraph-output.jsonata` - State-to-config transform + - `trustgraph-docs.yaml` - Documentation manifest + - `docs/` - Markdown documentation fragments + +## Error Handling + +The configurator includes error handling for: +- Missing or invalid templates +- Malformed input JSON +- File resolution failures +- Platform-specific generation errors + +Errors are logged with appropriate context for debugging. diff --git a/ai-context/trustgraph-templates/TEST_STRATEGY.md b/ai-context/trustgraph-templates/TEST_STRATEGY.md new file mode 100644 index 00000000..5896c3d5 --- /dev/null +++ b/ai-context/trustgraph-templates/TEST_STRATEGY.md @@ -0,0 +1,320 @@ +# TrustGraph Configurator Test Strategy + +## Executive Summary + +The TrustGraph configurator's primary risk is shipping broken configurations that fail to deploy properly. Unlike application crashes (which are obvious and easily fixable), configuration errors can cause silent failures, deployment issues, or runtime problems that are difficult to debug and significantly impact user experience. + +This test strategy prioritizes **configuration correctness** and **deployability** over code coverage metrics. + +## Risk Assessment + +### High Impact Risks +1. **Template Syntax Errors**: Jsonnet compilation failures that prevent configuration generation +2. **Invalid Configuration Structure**: Generated configs that don't match expected schemas +3. **Missing Dependencies**: Component configurations that reference non-existent services +4. **Platform-Specific Issues**: Configurations that work on one platform but fail on another +5. **Version Incompatibilities**: Template changes that break existing user configurations + +### Medium Impact Risks +1. **Performance Issues**: Inefficient template processing +2. **Resource Misconfigurations**: Incorrect memory/CPU limits +3. **Security Misconfigurations**: Missing security settings or exposed secrets + +### Low Impact Risks +1. **CLI Argument Parsing**: Easily debuggable and obvious failures +2. **Logging Issues**: Don't affect configuration correctness +3. **API Error Handling**: Obvious failures with clear error messages + +## Testing Strategy + +### 1. Configuration Validation Tests (Critical) + +#### Template Compilation Tests +- **Objective**: Ensure all Jsonnet templates compile without errors +- **Scope**: All templates in all versions (0.21, 0.22, 0.23, 1.0, 1.1) +- **Implementation**: + ```bash + # Test all platform configurations for each template version + for version in 0.21 0.22 0.23 1.0 1.1; do + for platform in docker-compose podman-compose minikube-k8s gcp-k8s aks-k8s eks-k8s scw-k8s; do + ./scripts/tg-build-deployment --template $version --platform $platform --input test-configs/minimal.json -O > /dev/null + done + done + ``` + +#### Schema Validation Tests +- **Objective**: Validate generated configurations against expected schemas +- **Docker Compose**: Validate against docker-compose schema +- **Kubernetes**: Validate against Kubernetes resource schemas +- **TrustGraph Config**: Validate against TrustGraph configuration schema + +#### Component Integration Tests +- **Objective**: Ensure component dependencies are correctly resolved +- **Test Cases**: + - LLM + Embeddings combinations (OpenAI + HF, Ollama + Ollama, etc.) + - RAG pipelines with all storage backends + - Graph databases with different embedding stores + - OCR + document processing chains + +### 2. Platform-Specific Deployment Tests (Critical) + +#### Docker Compose Validation +- **Objective**: Ensure generated docker-compose.yaml files are valid and deployable +- **Test Environment**: Local Docker daemon +- **Test Process**: + ```bash + # Generate configuration + ./scripts/tg-build-deployment --template 1.1 --platform docker-compose --input test-config.json -R > docker-compose.yaml + + # Validate syntax + docker-compose config -q + + # Test deployment (dry-run) + docker-compose up --no-start + + # Test actual deployment with minimal config + docker-compose up -d + timeout 60 docker-compose ps + docker-compose down + ``` + +#### Kubernetes Validation +- **Objective**: Ensure generated Kubernetes manifests are valid and deployable +- **Test Environment**: Minikube or kind cluster +- **Test Process**: + ```bash + # Generate configuration + ./scripts/tg-build-deployment --template 1.1 --platform gcp-k8s --input test-config.json -R > resources.yaml + + # Validate syntax + kubectl apply --dry-run=client -f resources.yaml + + # Test deployment + kubectl apply -f resources.yaml + kubectl wait --for=condition=Ready pod --all --timeout=300s + kubectl delete -f resources.yaml + ``` + +### 3. Configuration Matrix Tests (High Priority) + +#### Test Configuration Profiles +Create comprehensive test configurations covering: + +1. **Minimal Configuration** + ```json + { + "llm": {"engine": "openai", "model": "gpt-3.5-turbo"}, + "embeddings": {"engine": "hf", "model": "sentence-transformers/all-MiniLM-L6-v2"} + } + ``` + +2. **Complex RAG Configuration** + ```json + { + "llm": {"engine": "openai"}, + "embeddings": {"engine": "hf"}, + "vector_store": {"engine": "qdrant"}, + "graph_store": {"engine": "neo4j"}, + "document_store": {"engine": "cassandra"}, + "rag": {"enabled": true, "chunking": "recursive"} + } + ``` + +3. **Multi-Service Configuration** + ```json + { + "llm": [ + {"engine": "openai", "model": "gpt-4"}, + {"engine": "ollama", "model": "llama2"} + ], + "embeddings": {"engine": "fastembed"}, + "vector_store": {"engine": "milvus"}, + "monitoring": {"grafana": true, "prometheus": true} + } + ``` + +4. **Cloud-Specific Configurations** + - AWS Bedrock + EKS + - Azure OpenAI + AKS + - Google Vertex AI + GKE + +#### Cross-Platform Matrix +Test each configuration profile against all supported platforms: +- docker-compose +- podman-compose +- minikube-k8s +- gcp-k8s +- aks-k8s +- eks-k8s +- scw-k8s + +### 4. Version Compatibility Tests (Medium Priority) + +#### Backward Compatibility +- **Objective**: Ensure new template versions don't break existing configurations +- **Process**: + 1. Collect real-world configuration examples + 2. Test against new template versions + 3. Validate that outputs remain functionally equivalent + +#### Forward Compatibility +- **Objective**: Ensure template changes don't break when new features are added +- **Process**: + 1. Test configurations with unknown/future parameters + 2. Verify graceful degradation or clear error messages + +### 5. Integration Tests (Medium Priority) + +#### End-to-End Deployment Tests +- **Objective**: Verify complete deployment workflows +- **Test Cases**: + - Generate config β†’ Deploy β†’ Verify services start β†’ Run basic functionality test + - Test with monitoring enabled (Grafana/Prometheus accessible) + - Test with different storage backends + +#### Resource Generation Tests +- **Objective**: Verify auxiliary resources are correctly generated +- **Test Cases**: + - Grafana dashboards are valid JSON + - Prometheus config is valid YAML + - All required secrets/configmaps are created + +### 6. Regression Tests (High Priority) + +#### Template Change Validation +- **Objective**: Detect when template changes break existing configurations +- **Process**: + 1. Maintain golden configurations for each version + 2. Generate outputs before and after changes + 3. Compare outputs for breaking changes + 4. Flag any structural differences for manual review + +#### Component Regression Tests +- **Objective**: Ensure component updates don't break integrations +- **Test Cases**: + - Component parameter changes + - New component additions + - Component deprecations + +## Test Implementation Framework + +### Test Infrastructure + +#### Test Configuration Repository +``` +tests/ +β”œβ”€β”€ configs/ +β”‚ β”œβ”€β”€ minimal.json +β”‚ β”œβ”€β”€ complex-rag.json +β”‚ β”œβ”€β”€ multi-service.json +β”‚ └── cloud-specific/ +β”œβ”€β”€ golden/ +β”‚ β”œβ”€β”€ 1.1/ +β”‚ β”‚ β”œβ”€β”€ docker-compose/ +β”‚ β”‚ └── k8s/ +β”‚ └── 1.0/ +β”œβ”€β”€ schemas/ +β”‚ β”œβ”€β”€ docker-compose.json +β”‚ β”œβ”€β”€ k8s-resources.json +β”‚ └── trustgraph-config.json +└── scripts/ + β”œβ”€β”€ test-all-platforms.sh + β”œβ”€β”€ validate-schemas.sh + └── deploy-test.sh +``` + +#### Automated Test Pipeline +```bash +#!/bin/bash +# test-all-platforms.sh + +set -e + +VERSIONS="0.21 0.22 0.23 1.0 1.1" +PLATFORMS="docker-compose podman-compose minikube-k8s gcp-k8s aks-k8s eks-k8s scw-k8s" +CONFIGS="minimal.json complex-rag.json multi-service.json" + +echo "Testing configuration generation..." +for version in $VERSIONS; do + for platform in $PLATFORMS; do + for config in $CONFIGS; do + echo "Testing $version/$platform/$config" + + # Test TrustGraph config generation + ./scripts/tg-build-deployment --template $version --platform $platform \ + --input tests/configs/$config -O > /tmp/tg-config.json + + # Validate TrustGraph config + ./tests/scripts/validate-tg-config.sh /tmp/tg-config.json + + # Test resource generation + ./scripts/tg-build-deployment --template $version --platform $platform \ + --input tests/configs/$config -R > /tmp/resources.yaml + + # Validate resources + ./tests/scripts/validate-resources.sh /tmp/resources.yaml $platform + + echo "βœ“ $version/$platform/$config passed" + done + done +done +``` + +### Test Execution Strategy + +#### Continuous Integration +1. **Pre-commit**: Template syntax validation +2. **Pull Request**: Full configuration matrix tests +3. **Release**: Deployment tests + regression tests + +#### Performance Testing +- Template processing time benchmarks +- Memory usage during large configuration generation +- Concurrent API request handling + +#### Security Testing +- Validate no secrets are exposed in generated configs +- Test secret injection mechanisms +- Validate security-related component configurations + +## Test Metrics and Success Criteria + +### Primary Success Metrics +1. **Configuration Validity**: 100% of generated configurations must be syntactically valid +2. **Deployment Success**: 95% of generated configurations must deploy successfully +3. **Regression Prevention**: Zero breaking changes to existing configurations without explicit versioning + +### Secondary Success Metrics +1. **Test Coverage**: All template versions Γ— all platforms Γ— all major components +2. **Performance**: Configuration generation < 10 seconds for complex configs +3. **Documentation**: All test failures must have clear error messages and remediation steps + +## Maintenance and Updates + +### Test Maintenance +- Update test configurations when new features are added +- Refresh golden configurations when intentional changes are made +- Review and update schemas when component APIs change + +### Test Infrastructure Updates +- Add new platforms when supported +- Update validation tools when dependencies change +- Maintain test environments (Docker, Kubernetes clusters) + +## Risk Mitigation + +### High-Risk Changes +Any changes to the following require full test suite execution: +- Jsonnet template files +- Component definitions +- Engine implementations +- Version configuration changes + +### Emergency Procedures +- Rollback plan for broken template releases +- Hotfix process for critical configuration issues +- Communication plan for notifying users of breaking changes + +## Conclusion + +This test strategy prioritizes what matters most: ensuring users receive working, deployable configurations. By focusing on configuration correctness over code coverage, we can prevent the high-impact failures that would significantly affect user experience while maintaining development velocity for lower-risk changes. \ No newline at end of file diff --git a/ai-context/trustgraph-templates/config-standalone-pulsar.jsonnet b/ai-context/trustgraph-templates/config-standalone-pulsar.jsonnet new file mode 100644 index 00000000..0ab4d03a --- /dev/null +++ b/ai-context/trustgraph-templates/config-standalone-pulsar.jsonnet @@ -0,0 +1,45 @@ +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "object-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar-standalone", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "vertexai", + "parameters": { + "max-output-tokens": 8192 + } + } +] diff --git a/ai-context/trustgraph-templates/docs/garage-deployment.md b/ai-context/trustgraph-templates/docs/garage-deployment.md new file mode 100644 index 00000000..d00999bc --- /dev/null +++ b/ai-context/trustgraph-templates/docs/garage-deployment.md @@ -0,0 +1,244 @@ +# Garage S3-Compatible Object Storage + +## Overview + +Garage is a lightweight, self-hosted S3-compatible object storage system used as an alternative to Ceph for storing documents and other objects in the TrustGraph system. It provides a simpler deployment model while maintaining S3 API compatibility. + +## Features + +- **S3-Compatible API**: Full compatibility with AWS S3 SDK and CLI tools +- **Lightweight**: Minimal resource requirements compared to Ceph +- **Simple Deployment**: Single container with no complex dependencies +- **Distributed**: Supports multi-node clusters (though configured for single-node by default) + +## Architecture + +### Components + +1. **Garage Daemon** (`garage`) + - Main storage service container + - Provides S3 API on port 3900 + - Admin API on port 3903 + - RPC communication on port 3901 + +2. **Init Container** (`garage-init`) + - Runs once to initialize the cluster + - Configures cluster layout + - Creates S3 access credentials + - Uses remote RPC to communicate with daemon (no shared volumes) + +### Storage + +Garage uses two separate volumes: + +- **Metadata Volume** (`garage-meta`): Stores cluster metadata and LMDB database +- **Data Volume** (`garage-data`): Stores actual object data + +## Configuration Parameters + +All configuration is done via Jsonnet parameters with the `garage-` prefix: + +### S3 Credentials + +```jsonnet +"garage-access-key":: "GK000000000000000000000001", +"garage-secret-key":: "b171f00be9be4c32c734f4c05fe64c527a8ab5eb823b376cfa8c2531f70fc427", +``` + +**Format Requirements:** +- **Access Key ID**: Must start with `GK` followed by exactly 24 hex characters (0-9, a-f) +- **Secret Key**: Must be exactly 64 hex characters + +**Generate Secure Credentials:** + +```bash +# Generate Access Key ID (GK + 24 hex chars) +echo "GK$(openssl rand -hex 12)" + +# Generate Secret Key (64 hex chars) +openssl rand -hex 32 +``` + +### Cluster Configuration + +```jsonnet +"garage-rpc-secret":: "bbba746a9e289bad64a9e7a36a4299dac8d6e0b8cc2a6c2937fe756df4492008", +"garage-admin-token":: "batts-rockhearted-unpartially", +"garage-region":: "garage", +"garage-replication-factor":: "1", +``` + +- **rpc-secret**: 64 hex characters for node-to-node RPC authentication +- **admin-token**: Bearer token for Admin API access +- **region**: S3 region name +- **replication-factor**: Number of data replicas (set to 1 for single-node, 3+ for production) + +### Storage Volumes + +```jsonnet +"garage-meta-size":: "5G", +"garage-data-size":: "100G", +``` + +Both values can be overridden using the `.with()` function: + +```jsonnet +.with("meta-size", "10G") +.with("data-size", "500G") +``` + +## Integration with Librarian + +The librarian component automatically connects to Garage for object storage: + +```jsonnet +"--object-store-endpoint", url.object_store, +"--object-store-access-key", $["garage-access-key"], +"--object-store-secret-key", $["garage-secret-key"], +``` + +The object store endpoint is defined in `values/url.jsonnet`: + +```jsonnet +object_store: "http://garage:3900", +``` + +## Testing Garage with S3 Clients + +### Using AWS CLI + +```bash +# Set credentials +export AWS_ACCESS_KEY_ID="GK000000000000000000000001" +export AWS_SECRET_ACCESS_KEY="b171f00be9be4c32c734f4c05fe64c527a8ab5eb823b376cfa8c2531f70fc427" +export AWS_ENDPOINT_URL="http://localhost:3900" +export AWS_DEFAULT_REGION="garage" + +# Create a bucket +aws s3 mb s3://test-bucket + +# Upload a file +echo "Hello from Garage!" > test.txt +aws s3 cp test.txt s3://test-bucket/ + +# List files +aws s3 ls s3://test-bucket/ + +# Download a file +aws s3 cp s3://test-bucket/test.txt downloaded.txt + +# Verify +cat downloaded.txt +``` + +### Using s3cmd + +Create configuration file `~/.s3cfg`: + +```ini +[default] +access_key = GK000000000000000000000001 +secret_key = b171f00be9be4c32c734f4c05fe64c527a8ab5eb823b376cfa8c2531f70fc427 +host_base = localhost:3900 +host_bucket = localhost:3900 +use_https = False +``` + +Then use s3cmd: + +```bash +# List buckets +s3cmd ls + +# Create bucket +s3cmd mb s3://my-bucket + +# Upload file +s3cmd put file.txt s3://my-bucket/ + +# Download file +s3cmd get s3://my-bucket/file.txt +``` + +## Deployment Details + +### Initialization Process + +The init container performs these steps: + +1. **Wait for Garage daemon** - Polls `/health` endpoint until daemon is ready +2. **Get Node ID** - Queries `/v2/GetNodeInfo?node=self` via Admin API +3. **Configure Layout** - Assigns node to cluster with specified capacity via RPC +4. **Apply Layout** - Activates the cluster layout +5. **Import Credentials** - Creates S3 access key with provided credentials +6. **Grant Permissions** - Enables bucket creation for the key + +All operations are **idempotent** - the init container can be restarted safely and will skip already-configured items. + +### Network Architecture + +- **S3 API**: Port 3900 (HTTP) +- **RPC**: Port 3901 (internal cluster communication) +- **Web UI**: Port 3902 (optional web interface) +- **Admin API**: Port 3903 (cluster management) +- **K2V API**: Port 3904 (key-value store) + +### No Shared Volumes + +The init container communicates with the Garage daemon entirely over the network: + +- Admin API (HTTP) for status queries +- RPC (via garage CLI `-h` and `-s` flags) for cluster management + +This design works across all orchestrators (Kubernetes, Docker Compose) without requiring shared volume mounts. + +## Version Information + +Current deployment uses **Garage v2.1.0** + +- Image: `docker.io/dxflrs/garage:v2.1.0` +- Documentation: https://garagehq.deuxfleurs.fr/documentation/ + +## Troubleshooting + +### Init Container Fails + +Check logs: `podman logs ` + +Common issues: +- **403 Forbidden**: Check `garage-admin-token` is correct +- **Invalid layout version**: Cluster already initialized, init will retry +- **Node ID null**: Admin API not responding, check daemon logs + +### S3 Access Denied + +Verify credentials format: +- Access Key ID must start with `GK` + 24 hex chars +- Secret Key must be 64 hex chars +- Credentials must match what was imported during init + +### Check Garage Status + +```bash +# Query cluster status +curl -H "Authorization: Bearer " \ + http://localhost:3903/v2/GetClusterStatus + +# Check health +curl http://localhost:3903/health +``` + +## Production Considerations + +1. **Generate Secure Credentials**: Never use default credentials in production +2. **Set Replication Factor**: Use 3+ for redundancy in multi-node clusters +3. **Increase Storage Size**: Adjust `garage-data-size` based on expected usage +4. **Secure Admin Token**: Use a strong random token for `garage-admin-token` +5. **Monitor Storage**: Watch disk usage on data volume +6. **Backup Metadata**: The metadata volume contains critical cluster state + +## References + +- [Garage Documentation](https://garagehq.deuxfleurs.fr/documentation/) +- [Garage Admin API v2](https://garagehq.deuxfleurs.fr/api/garage-admin-v2.json) +- [Quick Start Guide](https://garagehq.deuxfleurs.fr/documentation/quick-start/) diff --git a/ai-context/trustgraph-templates/docs/tech-specs/tests.md b/ai-context/trustgraph-templates/docs/tech-specs/tests.md new file mode 100644 index 00000000..d695486a --- /dev/null +++ b/ai-context/trustgraph-templates/docs/tech-specs/tests.md @@ -0,0 +1,416 @@ +# Test Specification + +## Test Categories + +### Unit Tests +Test Python modules in isolation: +- **Generator** - Jsonnet template processing, import callbacks +- **Packager** - Zip file creation, configuration assembly +- **API** - Template listing, version resolution +- **CLI** - Argument parsing, error handling, exit codes + +### Integration Tests +Test full CLI workflow: +- Template compilation across version/platform/config matrix +- Output file generation (TrustGraph config + platform resources) +- Error propagation and reporting + +### Validation Tests +Verify correctness of generated outputs: +- Syntax validation (JSON/YAML parsing) +- Schema validation (structure compliance) +- Semantic validation (cross-references, consistency) +- Regression testing (golden files) + +## Test Matrix + +**Dimensions:** +- Versions: 1.6, 1.7, 1.8 +- Platforms: docker-compose, podman-compose, minikube-k8s, gcp-k8s, aks-k8s, eks-k8s, scw-k8s, ovh-k8s +- Configs: minimal.json, complex-rag.json, multi-service.json, cloud-aws.json + +**Total combinations:** 3 versions Γ— 8 platforms Γ— 4 configs = 96 combinations per output type = 192 tests + +## Validation Approaches + +### Syntax Validation +- **JSON**: Parse with `json.loads()`, check no exceptions +- **YAML**: Parse with `yaml.safe_load()`, check no exceptions +- **Docker Compose**: Validate with `docker-compose config` +- **Kubernetes**: Validate with `kubectl apply --dry-run=client` + +### Schema Validation +- **TrustGraph Config**: Define JSON schema, validate with `jsonschema` + - Required fields: services, modules, parameters + - Type checking for all configuration values + - Enum validation for fixed sets (llm providers, platforms) + +- **Kubernetes**: Check required fields + - apiVersion, kind, metadata present + - metadata.name, metadata.namespace defined + - spec structure matches resource kind + +- **Docker Compose**: Check required fields + - services defined with image/build + - Valid port mappings + - Valid volume definitions + +### Semantic Validation + +#### Kubernetes Resources +- **Label/Selector matching**: Deployment selectors match pod labels +- **Volume references**: volumeMounts reference defined volumes +- **Service targeting**: Service selectors match deployment labels +- **Port consistency**: containerPort matches service targetPort +- **ConfigMap/Secret references**: Referenced resources exist in manifest + +#### Docker Compose +- **Service dependencies**: depends_on references valid services +- **Volume references**: Volume names in bind mounts are defined +- **Network references**: Networks used by services are defined +- **Port conflicts**: No duplicate host port bindings +- **Environment variable references**: ${VAR} expansions are resolvable + +#### TrustGraph Config +- **Service references**: Configured services reference valid modules +- **Parameter validation**: Module parameters match schema +- **Storage consistency**: Graph/object/vector stores configured correctly +- **LLM configuration**: Valid model IDs, API configurations + +### Golden File Testing +Store reference outputs for each test case: +- **Location**: `tests/golden/{version}/{platform}/{config}/` +- **Files**: + - `tg-config.json` - Reference TrustGraph configuration + - `resources.yaml` - Reference platform resources +- **Comparison**: Use `pytest-golden` for automatic diff generation +- **Updates**: Explicit flag to regenerate golden files when intentional changes occur + +## Test Cases + +### Compilation Tests +For each (version, platform, config) combination: +1. Run `tg-build-deployment -t {version} -p {platform} -i {config} -O` +2. Assert exit code = 0 +3. Assert stdout contains valid JSON +4. Parse and validate TrustGraph config structure + +5. Run `tg-build-deployment -t {version} -p {platform} -i {config} -R` +6. Assert exit code = 0 +7. Assert stdout contains valid YAML +8. Parse and validate resource manifest structure + +### Error Handling Tests +- **Invalid config**: Malformed JSON input β†’ exit code 1, error to stderr +- **Missing file**: Non-existent config file β†’ exit code 1, error to stderr +- **Invalid template**: Non-existent version β†’ exit code 1, error to stderr +- **Invalid platform**: Non-existent platform β†’ exit code 1, error to stderr +- **Template errors**: Jsonnet compilation errors β†’ exit code 1, error to stderr + +### CLI Interface Tests +- **Argument parsing**: Valid/invalid argument combinations +- **Help output**: `-h` flag displays usage +- **Version display**: Version flag shows package version +- **Output modes**: `-O` and `-R` flags produce correct output types +- **Default values**: Missing optional args use documented defaults + +### Module Unit Tests + +#### Generator +- `process(config)` - Valid jsonnet β†’ parsed JSON +- `process(config)` - Invalid jsonnet β†’ raises exception +- Import callback mechanism works correctly +- Template loading from package resources + +#### Packager +- `write(config, output)` - Creates valid zip file +- `write_tg_config(config)` - Outputs TrustGraph config to stdout +- `write_resources(config)` - Outputs platform resources to stdout +- Template version resolution (--latest, --latest-stable) +- Platform-specific template selection + +## Test Execution Methods + +### Direct Function Call (Primary Method) +Most tests call the Python entry point function directly rather than invoking the subprocess: + +```python +from trustgraph_configurator import run +import sys +import json + +def test_basic_compilation(monkeypatch, capsys): + """Test compilation by calling run() directly""" + # Mock sys.argv with CLI arguments + monkeypatch.setattr(sys, 'argv', [ + 'tg-build-deployment', + '-t', '1.8', + '-p', 'docker-compose', + '-i', 'tests/configs/minimal.json', + '-O' + ]) + + # Call the entry point directly + run.run() + + # Capture and validate output + captured = capsys.readouterr() + config = json.loads(captured.out) + assert 'services' in config +``` + +**Advantages:** +- **Fast**: No subprocess overhead (100x+ faster) +- **Easy stdout/stderr capture**: Use pytest's `capsys` fixture +- **Easy mocking**: Use `monkeypatch` for arguments, environment, file system +- **Better debugging**: Direct code path, breakpoints work naturally +- **Exit code testing**: Catch `SystemExit` exception to verify exit codes + +```python +def test_error_handling(monkeypatch): + """Test that errors exit with code 1""" + monkeypatch.setattr(sys, 'argv', [ + 'tg-build-deployment', + '-i', 'nonexistent.json' + ]) + + with pytest.raises(SystemExit) as exc_info: + run.run() + + assert exc_info.value.code == 1 +``` + +### Subprocess Invocation (Smoke Tests) +A small number of tests (1-2) should invoke the actual CLI executable to verify installation: + +```python +import subprocess + +def test_cli_executable_installed(): + """Verify the installed CLI entry point works""" + result = subprocess.run( + ['tg-build-deployment', '--help'], + capture_output=True, + text=True + ) + assert result.returncode == 0 + assert 'usage:' in result.stdout + +def test_cli_version_command(): + """Verify version command works from CLI""" + result = subprocess.run( + ['tg-build-deployment', '--version'], + capture_output=True, + text=True + ) + assert result.returncode == 0 +``` + +**Purpose:** +- Verify `pyproject.toml` entry point configuration is correct +- Verify CLI is accessible in PATH after installation +- End-to-end smoke test + +**Limitations:** +- Slower (subprocess overhead) +- Harder to mock/patch +- Less detailed error information + +### Fixture for Direct Execution +Create a reusable fixture for calling configurator: + +```python +# tests/conftest.py +import pytest +import sys +from io import StringIO + +@pytest.fixture +def run_configurator(monkeypatch, capsys): + """Fixture to run configurator with given arguments""" + def _run(args): + """ + Run configurator with args list. + Returns (stdout, stderr, exit_code) + """ + from trustgraph_configurator import run + + monkeypatch.setattr(sys, 'argv', ['tg-build-deployment'] + args) + + exit_code = 0 + try: + run.run() + except SystemExit as e: + exit_code = e.code or 0 + + captured = capsys.readouterr() + return captured.out, captured.err, exit_code + + return _run +``` + +**Usage:** +```python +def test_with_fixture(run_configurator): + stdout, stderr, code = run_configurator([ + '-t', '1.8', + '-p', 'docker-compose', + '-i', 'tests/configs/minimal.json', + '-O' + ]) + assert code == 0 + assert json.loads(stdout) +``` + +## Test Infrastructure + +### Pytest Configuration +```toml +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--strict-markers", + "--cov=trustgraph_configurator", + "--cov-report=term-missing", + "--cov-report=html", +] +markers = [ + "unit: Unit tests", + "integration: Integration tests", + "validation: Output validation tests", + "slow: Slow-running tests", +] +``` + +### Fixtures (`tests/conftest.py`) +- `test_config_dir` - Path to tests/configs/ +- `test_configs` - Dict of loaded test configurations +- `temp_output_dir` - Temporary directory for test outputs +- `run_configurator` - Function to execute configurator CLI +- `golden_dir` - Path to golden file directory for test case + +### Parametrization +Use `pytest.mark.parametrize` for matrix testing: +```python +@pytest.mark.parametrize("version", ["1.6", "1.7", "1.8"]) +@pytest.mark.parametrize("platform", ["docker-compose", "minikube-k8s", ...]) +@pytest.mark.parametrize("config", ["minimal.json", "complex-rag.json", ...]) +def test_compilation(version, platform, config, run_configurator): + ... +``` + +### Parallel Execution +Use pytest-xdist for parallel test execution: +```bash +pytest -n auto # Use all CPU cores +``` + +## Test File Organization + +``` +tests/ +β”œβ”€β”€ conftest.py # Shared fixtures +β”œβ”€β”€ unit/ +β”‚ β”œβ”€β”€ test_generator.py +β”‚ β”œβ”€β”€ test_packager.py +β”‚ β”œβ”€β”€ test_api.py +β”‚ └── test_run.py +β”œβ”€β”€ integration/ +β”‚ β”œβ”€β”€ test_compilation.py # Template compilation matrix +β”‚ β”œβ”€β”€ test_cli.py # CLI interface tests +β”‚ └── test_errors.py # Error handling tests +β”œβ”€β”€ validation/ +β”‚ β”œβ”€β”€ test_syntax.py # Syntax validation +β”‚ β”œβ”€β”€ test_schema.py # Schema validation +β”‚ β”œβ”€β”€ test_semantics_k8s.py +β”‚ β”œβ”€β”€ test_semantics_docker.py +β”‚ └── test_semantics_tg.py +β”œβ”€β”€ configs/ # Test input configs (existing) +β”œβ”€β”€ schemas/ # JSON schemas for validation +β”‚ β”œβ”€β”€ trustgraph-config.schema.json +β”‚ β”œβ”€β”€ kubernetes-deployment.schema.json +β”‚ └── docker-compose.schema.json +β”œβ”€β”€ golden/ # Reference outputs +β”‚ └── {version}/{platform}/{config}/ +β”‚ β”œβ”€β”€ tg-config.json +β”‚ └── resources.yaml +└── validators/ # Validation helper modules + β”œβ”€β”€ kubernetes.py + β”œβ”€β”€ docker_compose.py + └── trustgraph.py +``` + +## Development Dependencies + +Add to pyproject.toml: +```toml +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-xdist>=3.0", # Parallel execution + "pytest-cov>=4.0", # Coverage reporting + "pytest-golden>=0.2", # Golden file testing + "jsonschema>=4.0", # Schema validation + "pyyaml>=6.0", # Already in main deps +] +``` + +Install for development: +```bash +pip install -e .[dev] +``` + +## CI/CD Integration + +Update `.github/workflows/pull-request.yaml`: +```yaml +- name: Install dependencies + run: | + python3 -m venv env + . env/bin/activate + pip install -e .[dev] + +- name: Run tests + run: | + . env/bin/activate + pytest -n auto --cov --cov-report=xml + +- name: Upload coverage + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml +``` + +## Running Tests + +```bash +# All tests +pytest + +# Specific category +pytest tests/unit/ +pytest tests/integration/ +pytest -m validation + +# Specific test file +pytest tests/unit/test_generator.py + +# Parallel execution +pytest -n auto + +# With coverage +pytest --cov=trustgraph_configurator --cov-report=html + +# Update golden files +pytest --update-golden + +# Verbose output +pytest -v + +# Stop on first failure +pytest -x +``` diff --git a/ai-context/trustgraph-templates/docs/templates-structure.md b/ai-context/trustgraph-templates/docs/templates-structure.md new file mode 100644 index 00000000..95bb04b8 --- /dev/null +++ b/ai-context/trustgraph-templates/docs/templates-structure.md @@ -0,0 +1,188 @@ +# How TrustGraph Templates Are Structured + +## Overview + +The TrustGraph template system is a Jsonnet-based configuration framework that generates deployment configurations for multiple platforms (Docker Compose, Kubernetes, etc.) from a single JSON configuration file. The system uses a component-based architecture with abstraction layers for different deployment targets. + +## Directory Structure + +``` +trustgraph_configurator/ +β”œβ”€β”€ packager.py # Main entry point +β”œβ”€β”€ generator.py # Jsonnet processor wrapper +β”œβ”€β”€ templates/ +β”‚ └── 1.3/ # Template version +β”‚ β”œβ”€β”€ components.jsonnet # Component registry +β”‚ β”œβ”€β”€ config-to-docker-compose.jsonnet # Docker Compose generator +β”‚ β”œβ”€β”€ config-to-tg-configuration.jsonnet # TrustGraph config extractor +β”‚ β”œβ”€β”€ components/ # Component definitions +β”‚ β”œβ”€β”€ engine/ # Platform-specific engines +β”‚ β”œβ”€β”€ util/ # Utility functions +β”‚ β”œβ”€β”€ prompts/ # LLM prompt templates +β”‚ └── values/ # Shared configuration values +└── resources/ + └── 1.3/ # Static resource files + β”œβ”€β”€ grafana/ # Grafana dashboards/configs + └── prometheus/ # Prometheus configs +``` + +## Core Concepts + +### 1. Components + +Components are the building blocks of the system. Each component: +- Defines configuration parameters with defaults +- Implements a `create` function that generates platform-specific resources +- Can compose with other components through Jsonnet's object composition (`+`) +- Lives in `components/` directory + +Example component structure (simplified `ollama.jsonnet`): +```jsonnet +{ + // Parameter with default value + "ollama-model":: "gemma2:9b", + + // Service definition + "text-completion" +: { + create:: function(engine) + // Use engine abstraction to create resources + local container = engine.container("text-completion") + .with_image(...) + .with_command([...]); + + engine.resources([container, ...]) + }, + + // Custom parameter setter + with:: function(key, value) + self + { ["ollama-" + key]:: value } +} +``` + +### 2. Engines + +Engines provide platform-specific implementations for resource creation. Each engine implements: +- `container()` - Create container definitions +- `service()` - Create service definitions +- `volume()` - Create volume definitions +- `resources()` - Aggregate resources for output + +The engine abstraction allows components to be platform-agnostic. Available engines: +- `docker-compose.jsonnet` - Docker Compose format +- `noop.jsonnet` - No-op engine for configuration extraction only +- Kubernetes engines (various cloud providers) + +### 3. Configuration Flow + +``` +config.json β†’ decode β†’ patterns β†’ engine.create() β†’ resources β†’ output +``` + +1. **Input**: `config.json` contains a list of components to enable: +```json +[ + {"name": "trustgraph-base", "parameters": {}}, + {"name": "ollama", "parameters": {"model": "mixtral"}} +] +``` + +2. **Decode**: The `decode-config.jsonnet` utility: + - Loads each component from the registry + - Applies parameters using the `with_params` function + - Merges all components into a single "patterns" object + +3. **Engine Processing**: Platform-specific files like `config-to-docker-compose.jsonnet`: + - Call each component's `create(engine)` function + - Fold results into final resource structure + - Output platform-specific configuration + +### 4. Configuration Extraction + +The `config-to-tg-configuration.jsonnet` file extracts the TrustGraph runtime configuration from components. This includes: +- Prompt templates +- Flow definitions +- Model token costs +- Agent tools +- MCP server configurations + +This configuration is embedded into the deployment separately from the infrastructure resources. + +## Processing Pipeline + +### Entry Point: `packager.py` + +The Packager class orchestrates the entire process: + +1. **Template Selection**: Determines version and template based on user input +2. **Resource Generation**: + - For Docker Compose: Calls `config-to-docker-compose.jsonnet` + - For Kubernetes: Calls `config-to--k8s.jsonnet` +3. **Configuration Generation**: Calls `config-to-tg-configuration.jsonnet` for runtime config +4. **Packaging**: Creates ZIP file with all generated files and static resources + +### Jsonnet Processing: `generator.py` + +Simple wrapper around the Jsonnet library that: +- Evaluates Jsonnet templates +- Provides custom import callback for file resolution +- Returns parsed JSON output + +## Component Composition + +Components can be composed in several ways: + +### 1. Base Component Extension +Many components extend `trustgraph-base` which provides core services: +```jsonnet +local trustgraph = import "components/trustgraph.jsonnet"; +// Inherits all trustgraph services +{} + trustgraph + myCustomizations +``` + +### 2. Field Merging +Components use `+:` to merge with existing fields: +```jsonnet +"text-completion" +: { + // Adds to existing text-completion definition + create:: function(engine) ... +} +``` + +### 3. Parameter Injection +The `with` pattern allows runtime parameter injection: +```jsonnet +with:: function(key, value) + self + { [key]:: value } +``` + +## Hidden Fields and Configuration + +Jsonnet's `::` operator creates hidden fields that aren't included in JSON output by default. The template system uses this for: +- Default values that can be overridden +- Internal helper functions +- Configuration that needs special extraction + +For example, `trustgraph-base` has a hidden `configuration::` field containing runtime config that's extracted separately by `config-to-tg-configuration.jsonnet`. + +## Platform Abstraction + +The engine pattern provides clean separation between: +- **Component logic** - What services/containers to create +- **Platform specifics** - How to represent them (Docker Compose YAML, K8s manifests, etc.) + +This allows the same component definitions to generate configurations for multiple platforms without modification. + +## Static Resources + +Files in `resources/` are copied directly to the output package. These include: +- Grafana dashboard definitions +- Prometheus configuration +- Other platform-specific configs that don't need templating + +## Best Practices + +1. **Component Independence**: Components should be self-contained and not depend on specific ordering +2. **Parameter Namespacing**: Use prefixes (e.g., `ollama-model`) to avoid conflicts +3. **Hidden Fields for Defaults**: Use `::` for overridable defaults +4. **Engine Abstraction**: Always use engine methods rather than creating platform-specific structures directly +5. **Composition Over Inheritance**: Use Jsonnet's object composition (`+`) rather than complex inheritance hierarchies \ No newline at end of file diff --git a/ai-context/trustgraph-templates/examples/intel-battlemage-vllm.json b/ai-context/trustgraph-templates/examples/intel-battlemage-vllm.json new file mode 100644 index 00000000..fd7f8a6c --- /dev/null +++ b/ai-context/trustgraph-templates/examples/intel-battlemage-vllm.json @@ -0,0 +1,59 @@ +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "object-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": { + "text-completion-concurrency": 16, + "text-completion-rag-concurrency": 16, + "prompt-concurrency": 16, + "prompt-rag-concurrency": 16, + "kg-extraction-concurrency": 16, + "embeddings-concurrency": 16, + } + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": {} + }, + { + "name": "vllm", + "parameters": { + "max-output-tokens": 8192 + } + }, + { + "name": "hosting-intel-battlemage-vllm", + "parameters": { + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "cpus": "32.0", + "memory": "48G", + "hf-token": "", + } + }, +] diff --git a/ai-context/trustgraph-templates/pulumi/Pulumi.prod.yaml b/ai-context/trustgraph-templates/pulumi/Pulumi.prod.yaml new file mode 100644 index 00000000..cc14f03c --- /dev/null +++ b/ai-context/trustgraph-templates/pulumi/Pulumi.prod.yaml @@ -0,0 +1,14 @@ +encryptionsalt: v1:vQGk98eEeYI=:v1:tHg+f1b66tEydgA9:J1RGVNI0FssyjSXVhcKU7bfBofNFTg== +config: + config-svc:artifact-name: config-svc + config-svc:artifact-repo: europe-west1-docker.pkg.dev/trustgraph-ai/config-svc + config-svc:artifact-repo-region: europe-west1 + config-svc:cloud-run-region: europe-west1 + config-svc:domain: app.trustgraph.ai + config-svc:environment: prod + config-svc:gcp-project: trustgraph-ai + config-svc:gcp-region: europe-west1 + config-svc:hostname: config-svc.app.trustgraph.ai + config-svc:managed-zone: app + config-svc:max-scale: "1" + config-svc:min-scale: "0" diff --git a/ai-context/trustgraph-templates/pulumi/Pulumi.yaml b/ai-context/trustgraph-templates/pulumi/Pulumi.yaml new file mode 100644 index 00000000..9898a5b1 --- /dev/null +++ b/ai-context/trustgraph-templates/pulumi/Pulumi.yaml @@ -0,0 +1,3 @@ +name: config-svc +runtime: nodejs +description: Config service diff --git a/ai-context/trustgraph-templates/pulumi/index.ts b/ai-context/trustgraph-templates/pulumi/index.ts new file mode 100644 index 00000000..394dfd68 --- /dev/null +++ b/ai-context/trustgraph-templates/pulumi/index.ts @@ -0,0 +1,323 @@ + +import * as pulumi from "@pulumi/pulumi"; +import * as gcp from "@pulumi/gcp"; +import { local } from "@pulumi/command"; +import * as fs from 'fs'; + +const cfg = new pulumi.Config(); + +function get(tag : string) { + + let val = cfg.get(tag); + + if (!val) { + console.log("ERROR: The '" + tag + "' config is mandatory"); + throw "The '" + tag + "' config is mandatory"; + } + + return val; + +} + +const imageVersion = process.env.IMAGE_VERSION; +if (!imageVersion) + throw Error("IMAGE_VERSION not defined"); + +const repo = get("artifact-repo"); +const artifactRepoRegion = get("artifact-repo-region"); +const artifactName = get("artifact-name"); +const hostname = get("hostname"); +const managedZone = get("managed-zone"); +const project = get("gcp-project"); +const region = get("gcp-region"); +const cloudRunRegion = get("cloud-run-region"); +const environment = get("environment"); +const domain = get("domain"); +const minScale = get("min-scale"); +const maxScale = get("max-scale"); + +const provider = new gcp.Provider( + "gcp", + { + project: project, + region: region, + } +); + +const artifactRepo = new gcp.artifactregistry.Repository( + "artifact-repo", + { + description: "repository for " + environment, + format: "DOCKER", + location: artifactRepoRegion, + repositoryId: artifactName, + cleanupPolicies: [ + { + id: "keep-minimum-versions", + action: "KEEP", + mostRecentVersions: { + keepCount: 5, + }, + } + ], + }, + { + provider: provider, + } +); + +const localImageName = "localhost/config-svc:" + imageVersion; + +const imageName = repo + "/config-svc:" + imageVersion; + +const taggedImage = new local.Command( + "podman-tag-command", + { + create: "podman tag " + localImageName + " " + imageName, + } +); + +const image = new local.Command( + "podman-push-command", + { + create: "podman push " + imageName, + }, + { + dependsOn: [taggedImage, artifactRepo], + } +); + +const svcAccount = new gcp.serviceaccount.Account( + "service-account", + { + accountId: "config-svc-" + environment, + displayName: "Config service", + description: "Config service", + }, + { + provider: provider, + } +); + +const service = new gcp.cloudrun.Service( + "service", + { + name: "config-svc-" + environment, + location: cloudRunRegion, + template: { + metadata: { + labels: { + version: "v" + imageVersion.replace(/\./g, "-"), + }, + annotations: { + + // Scale attributes + "autoscaling.knative.dev/minScale": minScale, + "autoscaling.knative.dev/maxScale": maxScale, + + // 2nd generation. Need to specify at least 512MB RAM. + // Going back to gen1 because faster cold starts + "run.googleapis.com/execution-environment": "gen1", + + } + }, + spec: { + containerConcurrency: 100, + timeoutSeconds: 300, + serviceAccountName: svcAccount.email, + containers: [ + { + image: imageName, + ports: [ + { + "name": "http1", // Must be http1 or h2c. + "containerPort": 8080, + } + ], + resources: { + limits: { + cpu: "1000m", + memory: "512Mi", + } + }, + } + ], + }, + }, + }, + { + provider: provider, + dependsOn: [image], + } +); + +const allUsersPolicy = gcp.organizations.getIAMPolicy( + { + bindings: [{ + role: "roles/run.invoker", + members: ["allUsers"], + }], + }, + { + provider: provider, + } +); + +const noAuthPolicy = new gcp.cloudrun.IamPolicy( + "no-auth-policy", + { + location: service.location, + project: service.project, + service: service.name, + policyData: allUsersPolicy.then(pol => pol.policyData), + }, + { + provider: provider, + } +); + +//////////////////////////////////////////////////////////////////////////// + +const domainMapping = new gcp.cloudrun.DomainMapping( + "domain-mapping", + { + name: hostname, + location: cloudRunRegion, + metadata: { + namespace: project, + }, + spec: { + routeName: service.name, + } + }, + { + provider: provider + } +); + +const zone = gcp.dns.getManagedZoneOutput( + { + name: managedZone, + }, + { + provider: provider, + } +); + +//////////////////////////////////////////////////////////////////////////// + +domainMapping.statuses.apply( + ss => ss[0].resourceRecords +).apply( + rrs => { + if (rrs) { + + let mapping : { [k : string] : string[] } = {}; + + for(var i = 0; i < rrs.length; i++) { + if (rrs[i].rrdata) { + + const rr = rrs[i].rrdata; + const tp = rrs[i].type; + + if (!rr || !tp) continue; + + if (mapping[tp]) + mapping[tp].push(rr); + else + mapping[tp] = [rr]; + + } + } + + for (let tp in mapping) { + + const recordSet = new gcp.dns.RecordSet( + "resource-record-" + tp, + { + name: hostname + ".", + managedZone: zone.name, + type: tp, + ttl: 300, + rrdatas: mapping[tp], + }, + { + provider: provider, + } + ); + + } + + } + + } +); + +//////////////////////////////////////////////////////////////////////////// + +const serviceMon = new gcp.monitoring.GenericService( + "service-monitoring", + { + basicService: { + serviceLabels: { + service_name: service.name, + location: cloudRunRegion, + }, + serviceType: "CLOUD_RUN", + }, + displayName: "Config service (" + environment + ")", + serviceId: "config-service-" + environment + "-mon", + userLabels: { + "service": service.name, + "application": "config-svc", + "environment": environment, + }, + }, + { + provider: provider, + } +); + +const latencySlo = new gcp.monitoring.Slo( + "latency-slo", + { + service: serviceMon.serviceId, + sloId: "config-service-" + environment + "-latency-slo", + displayName: "Config service latency (" + environment + ")", + goal: 0.95, + rollingPeriodDays: 5, + basicSli: { + latency: { + threshold: "2s" + } + }, + }, + { + provider: provider, + } +); + +const availabilitySlo = new gcp.monitoring.Slo( + "availability-slo", + { + service: serviceMon.serviceId, + sloId: "config-service-" + environment + "-availability-slo", + displayName: "Config service availability (" + environment + ")", + goal: 0.95, + rollingPeriodDays: 5, + windowsBasedSli: { + windowPeriod: "3600s", + goodTotalRatioThreshold: { + basicSliPerformance: { + availability: { + } + }, + threshold: 0.9, + } + } + }, + { + provider: provider, + } +); + diff --git a/ai-context/trustgraph-templates/pulumi/package-lock.json b/ai-context/trustgraph-templates/pulumi/package-lock.json new file mode 100644 index 00000000..6cd217dd --- /dev/null +++ b/ai-context/trustgraph-templates/pulumi/package-lock.json @@ -0,0 +1,4695 @@ +{ + "name": "pulumi", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "pulumi", + "dependencies": { + "@pulumi/command": "^1.1.3", + "@pulumi/gcp": "^9.10.0", + "@pulumi/pulumi": "^3.216.0" + } + }, + "node_modules/@gar/promise-retry": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@gar/promise-retry/-/promise-retry-1.0.2.tgz", + "integrity": "sha512-Lm/ZLhDZcBECta3TmCQSngiQykFdfw+QtI1/GYMsZd4l3nG+P8WLB16XuS7WaBGLQ+9E+cOcWQsth9cayuGt8g==", + "license": "MIT", + "dependencies": { + "retry": "^0.13.1" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", + "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz", + "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.3", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@isaacs/string-locale-compare": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@isaacs/string-locale-compare/-/string-locale-compare-1.1.0.tgz", + "integrity": "sha512-SQ7Kzhh9+D+ZW9MA0zkYv3VXhIDNx+LzM6EJ+/65I3QY+enU6Itte7E5XX7EWrqLW2FN4n06GWzBnPoC3th2aQ==", + "license": "ISC" + }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@logdna/tail-file": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@logdna/tail-file/-/tail-file-2.2.0.tgz", + "integrity": "sha512-XGSsWDweP80Fks16lwkAUIr54ICyBs6PsI4mpfTLQaWgEJRtY9xEV+PeyDpJ+sJEGZxqINlpmAwe/6tS1pP8Ng==", + "license": "SEE LICENSE IN LICENSE", + "engines": { + "node": ">=10.3.0" + } + }, + "node_modules/@npmcli/agent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-4.0.0.tgz", + "integrity": "sha512-kAQTcEN9E8ERLVg5AsGwLNoFb+oEG6engbqAU2P43gD4JEIkNGMHdVQ096FsOAAYpZPB0RSt0zgInKIAS1l5QA==", + "license": "ISC", + "dependencies": { + "agent-base": "^7.1.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", + "lru-cache": "^11.2.1", + "socks-proxy-agent": "^8.0.3" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/agent/node_modules/lru-cache": { + "version": "11.2.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", + "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@npmcli/arborist": { + "version": "9.4.0", + "resolved": "https://registry.npmjs.org/@npmcli/arborist/-/arborist-9.4.0.tgz", + "integrity": "sha512-4Bm8hNixJG/sii1PMnag0V9i/sGOX9VRzFrUiZMSBJpGlLR38f+Btl85d07G9GL56xO0l0OZjvrGNYsDYp0xKA==", + "license": "ISC", + "dependencies": { + "@isaacs/string-locale-compare": "^1.1.0", + "@npmcli/fs": "^5.0.0", + "@npmcli/installed-package-contents": "^4.0.0", + "@npmcli/map-workspaces": "^5.0.0", + "@npmcli/metavuln-calculator": "^9.0.2", + "@npmcli/name-from-folder": "^4.0.0", + "@npmcli/node-gyp": "^5.0.0", + "@npmcli/package-json": "^7.0.0", + "@npmcli/query": "^5.0.0", + "@npmcli/redact": "^4.0.0", + "@npmcli/run-script": "^10.0.0", + "bin-links": "^6.0.0", + "cacache": "^20.0.1", + "common-ancestor-path": "^2.0.0", + "hosted-git-info": "^9.0.0", + "json-stringify-nice": "^1.1.4", + "lru-cache": "^11.2.1", + "minimatch": "^10.0.3", + "nopt": "^9.0.0", + "npm-install-checks": "^8.0.0", + "npm-package-arg": "^13.0.0", + "npm-pick-manifest": "^11.0.1", + "npm-registry-fetch": "^19.0.0", + "pacote": "^21.0.2", + "parse-conflict-json": "^5.0.1", + "proc-log": "^6.0.0", + "proggy": "^4.0.0", + "promise-all-reject-late": "^1.0.0", + "promise-call-limit": "^3.0.1", + "semver": "^7.3.7", + "ssri": "^13.0.0", + "treeverse": "^3.0.0", + "walk-up-path": "^4.0.0" + }, + "bin": { + "arborist": "bin/index.js" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/arborist/node_modules/@npmcli/git": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@npmcli/git/-/git-7.0.2.tgz", + "integrity": "sha512-oeolHDjExNAJAnlYP2qzNjMX/Xi9bmu78C9dIGr4xjobrSKbuMYCph8lTzn4vnW3NjIqVmw/f8BCfouqyJXlRg==", + "license": "ISC", + "dependencies": { + "@gar/promise-retry": "^1.0.0", + "@npmcli/promise-spawn": "^9.0.0", + "ini": "^6.0.0", + "lru-cache": "^11.2.1", + "npm-pick-manifest": "^11.0.1", + "proc-log": "^6.0.0", + "semver": "^7.3.5", + "which": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/arborist/node_modules/@npmcli/package-json": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@npmcli/package-json/-/package-json-7.0.5.tgz", + "integrity": "sha512-iVuTlG3ORq2iaVa1IWUxAO/jIp77tUKBhoMjuzYW2kL4MLN1bi/ofqkZ7D7OOwh8coAx1/S2ge0rMdGv8sLSOQ==", + "license": "ISC", + "dependencies": { + "@npmcli/git": "^7.0.0", + "glob": "^13.0.0", + "hosted-git-info": "^9.0.0", + "json-parse-even-better-errors": "^5.0.0", + "proc-log": "^6.0.0", + "semver": "^7.5.3", + "spdx-expression-parse": "^4.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/arborist/node_modules/@npmcli/promise-spawn": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-9.0.1.tgz", + "integrity": "sha512-OLUaoqBuyxeTqUvjA3FZFiXUfYC1alp3Sa99gW3EUDz3tZ3CbXDdcZ7qWKBzicrJleIgucoWamWH1saAmH/l2Q==", + "license": "ISC", + "dependencies": { + "which": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/arborist/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@npmcli/arborist/node_modules/hosted-git-info": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.2.tgz", + "integrity": "sha512-M422h7o/BR3rmCQ8UHi7cyyMqKltdP9Uo+J2fXK+RSAY+wTcKOIRyhTuKv4qn+DJf3g+PL890AzId5KZpX+CBg==", + "license": "ISC", + "dependencies": { + "lru-cache": "^11.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/arborist/node_modules/ini": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-6.0.0.tgz", + "integrity": "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/arborist/node_modules/isexe": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", + "integrity": "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=20" + } + }, + "node_modules/@npmcli/arborist/node_modules/json-parse-even-better-errors": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-5.0.0.tgz", + "integrity": "sha512-ZF1nxZ28VhQouRWhUcVlUIN3qwSgPuswK05s/HIaoetAoE/9tngVmCHjSxmSQPav1nd+lPtTL0YZ/2AFdR/iYQ==", + "license": "MIT", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/arborist/node_modules/lru-cache": { + "version": "11.2.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", + "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@npmcli/arborist/node_modules/npm-pick-manifest": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-11.0.3.tgz", + "integrity": "sha512-buzyCfeoGY/PxKqmBqn1IUJrZnUi1VVJTdSSRPGI60tJdUhUoSQFhs0zycJokDdOznQentgrpf8LayEHyyYlqQ==", + "license": "ISC", + "dependencies": { + "npm-install-checks": "^8.0.0", + "npm-normalize-package-bin": "^5.0.0", + "npm-package-arg": "^13.0.0", + "semver": "^7.3.5" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/arborist/node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@npmcli/arborist/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/arborist/node_modules/spdx-expression-parse": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-4.0.0.tgz", + "integrity": "sha512-Clya5JIij/7C6bRR22+tnGXbc4VKlibKSVj2iHvVeX5iMW7s1SIQlqu699JkODJJIhh/pUu8L0/VLh8xflD+LQ==", + "license": "MIT", + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/@npmcli/arborist/node_modules/which": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", + "integrity": "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==", + "license": "ISC", + "dependencies": { + "isexe": "^4.0.0" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/fs": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-5.0.0.tgz", + "integrity": "sha512-7OsC1gNORBEawOa5+j2pXN9vsicaIOH5cPXxoR6fJOmH6/EXpJB2CajXOu1fPRFun2m1lktEFX11+P89hqO/og==", + "license": "ISC", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/git": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@npmcli/git/-/git-6.0.3.tgz", + "integrity": "sha512-GUYESQlxZRAdhs3UhbB6pVRNUELQOHXwK9ruDkwmCv2aZ5y0SApQzUJCg02p3A7Ue2J5hxvlk1YI53c00NmRyQ==", + "license": "ISC", + "dependencies": { + "@npmcli/promise-spawn": "^8.0.0", + "ini": "^5.0.0", + "lru-cache": "^10.0.1", + "npm-pick-manifest": "^10.0.0", + "proc-log": "^5.0.0", + "promise-retry": "^2.0.1", + "semver": "^7.3.5", + "which": "^5.0.0" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/@npmcli/installed-package-contents": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@npmcli/installed-package-contents/-/installed-package-contents-4.0.0.tgz", + "integrity": "sha512-yNyAdkBxB72gtZ4GrwXCM0ZUedo9nIbOMKfGjt6Cu6DXf0p8y1PViZAKDC8q8kv/fufx0WTjRBdSlyrvnP7hmA==", + "license": "ISC", + "dependencies": { + "npm-bundled": "^5.0.0", + "npm-normalize-package-bin": "^5.0.0" + }, + "bin": { + "installed-package-contents": "bin/index.js" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/map-workspaces": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/@npmcli/map-workspaces/-/map-workspaces-5.0.3.tgz", + "integrity": "sha512-o2grssXo1e774E5OtEwwrgoszYRh0lqkJH+Pb9r78UcqdGJRDRfhpM8DvZPjzNLLNYeD/rNbjOKM3Ss5UABROw==", + "license": "ISC", + "dependencies": { + "@npmcli/name-from-folder": "^4.0.0", + "@npmcli/package-json": "^7.0.0", + "glob": "^13.0.0", + "minimatch": "^10.0.3" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/@npmcli/git": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@npmcli/git/-/git-7.0.2.tgz", + "integrity": "sha512-oeolHDjExNAJAnlYP2qzNjMX/Xi9bmu78C9dIGr4xjobrSKbuMYCph8lTzn4vnW3NjIqVmw/f8BCfouqyJXlRg==", + "license": "ISC", + "dependencies": { + "@gar/promise-retry": "^1.0.0", + "@npmcli/promise-spawn": "^9.0.0", + "ini": "^6.0.0", + "lru-cache": "^11.2.1", + "npm-pick-manifest": "^11.0.1", + "proc-log": "^6.0.0", + "semver": "^7.3.5", + "which": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/@npmcli/package-json": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@npmcli/package-json/-/package-json-7.0.5.tgz", + "integrity": "sha512-iVuTlG3ORq2iaVa1IWUxAO/jIp77tUKBhoMjuzYW2kL4MLN1bi/ofqkZ7D7OOwh8coAx1/S2ge0rMdGv8sLSOQ==", + "license": "ISC", + "dependencies": { + "@npmcli/git": "^7.0.0", + "glob": "^13.0.0", + "hosted-git-info": "^9.0.0", + "json-parse-even-better-errors": "^5.0.0", + "proc-log": "^6.0.0", + "semver": "^7.5.3", + "spdx-expression-parse": "^4.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/@npmcli/promise-spawn": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-9.0.1.tgz", + "integrity": "sha512-OLUaoqBuyxeTqUvjA3FZFiXUfYC1alp3Sa99gW3EUDz3tZ3CbXDdcZ7qWKBzicrJleIgucoWamWH1saAmH/l2Q==", + "license": "ISC", + "dependencies": { + "which": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/hosted-git-info": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.2.tgz", + "integrity": "sha512-M422h7o/BR3rmCQ8UHi7cyyMqKltdP9Uo+J2fXK+RSAY+wTcKOIRyhTuKv4qn+DJf3g+PL890AzId5KZpX+CBg==", + "license": "ISC", + "dependencies": { + "lru-cache": "^11.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/ini": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-6.0.0.tgz", + "integrity": "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/isexe": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", + "integrity": "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=20" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/json-parse-even-better-errors": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-5.0.0.tgz", + "integrity": "sha512-ZF1nxZ28VhQouRWhUcVlUIN3qwSgPuswK05s/HIaoetAoE/9tngVmCHjSxmSQPav1nd+lPtTL0YZ/2AFdR/iYQ==", + "license": "MIT", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/lru-cache": { + "version": "11.2.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", + "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/npm-pick-manifest": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-11.0.3.tgz", + "integrity": "sha512-buzyCfeoGY/PxKqmBqn1IUJrZnUi1VVJTdSSRPGI60tJdUhUoSQFhs0zycJokDdOznQentgrpf8LayEHyyYlqQ==", + "license": "ISC", + "dependencies": { + "npm-install-checks": "^8.0.0", + "npm-normalize-package-bin": "^5.0.0", + "npm-package-arg": "^13.0.0", + "semver": "^7.3.5" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/spdx-expression-parse": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-4.0.0.tgz", + "integrity": "sha512-Clya5JIij/7C6bRR22+tnGXbc4VKlibKSVj2iHvVeX5iMW7s1SIQlqu699JkODJJIhh/pUu8L0/VLh8xflD+LQ==", + "license": "MIT", + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/@npmcli/map-workspaces/node_modules/which": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", + "integrity": "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==", + "license": "ISC", + "dependencies": { + "isexe": "^4.0.0" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/metavuln-calculator": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/@npmcli/metavuln-calculator/-/metavuln-calculator-9.0.3.tgz", + "integrity": "sha512-94GLSYhLXF2t2LAC7pDwLaM4uCARzxShyAQKsirmlNcpidH89VA4/+K1LbJmRMgz5gy65E/QBBWQdUvGLe2Frg==", + "license": "ISC", + "dependencies": { + "cacache": "^20.0.0", + "json-parse-even-better-errors": "^5.0.0", + "pacote": "^21.0.0", + "proc-log": "^6.0.0", + "semver": "^7.3.5" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/metavuln-calculator/node_modules/json-parse-even-better-errors": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-5.0.0.tgz", + "integrity": "sha512-ZF1nxZ28VhQouRWhUcVlUIN3qwSgPuswK05s/HIaoetAoE/9tngVmCHjSxmSQPav1nd+lPtTL0YZ/2AFdR/iYQ==", + "license": "MIT", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/metavuln-calculator/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/name-from-folder": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@npmcli/name-from-folder/-/name-from-folder-4.0.0.tgz", + "integrity": "sha512-qfrhVlOSqmKM8i6rkNdZzABj8MKEITGFAY+4teqBziksCQAOLutiAxM1wY2BKEd8KjUSpWmWCYxvXr0y4VTlPg==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/node-gyp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@npmcli/node-gyp/-/node-gyp-5.0.0.tgz", + "integrity": "sha512-uuG5HZFXLfyFKqg8QypsmgLQW7smiRjVc45bqD/ofZZcR/uxEjgQU8qDPv0s9TEeMUiAAU/GC5bR6++UdTirIQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/package-json": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/@npmcli/package-json/-/package-json-6.2.0.tgz", + "integrity": "sha512-rCNLSB/JzNvot0SEyXqWZ7tX2B5dD2a1br2Dp0vSYVo5jh8Z0EZ7lS9TsZ1UtziddB1UfNUaMCc538/HztnJGA==", + "license": "ISC", + "dependencies": { + "@npmcli/git": "^6.0.0", + "glob": "^10.2.2", + "hosted-git-info": "^8.0.0", + "json-parse-even-better-errors": "^4.0.0", + "proc-log": "^5.0.0", + "semver": "^7.5.3", + "validate-npm-package-license": "^3.0.4" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/@npmcli/promise-spawn": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-8.0.3.tgz", + "integrity": "sha512-Yb00SWaL4F8w+K8YGhQ55+xE4RUNdMHV43WZGsiTM92gS+lC0mGsn7I4hLug7pbao035S6bj3Y3w0cUNGLfmkg==", + "license": "ISC", + "dependencies": { + "which": "^5.0.0" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/@npmcli/query": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@npmcli/query/-/query-5.0.0.tgz", + "integrity": "sha512-8TZWfTQOsODpLqo9SVhVjHovmKXNpevHU0gO9e+y4V4fRIOneiXy0u0sMP9LmS71XivrEWfZWg50ReH4WRT4aQ==", + "license": "ISC", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/redact": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@npmcli/redact/-/redact-4.0.0.tgz", + "integrity": "sha512-gOBg5YHMfZy+TfHArfVogwgfBeQnKbbGo3pSUyK/gSI0AVu+pEiDVcKlQb0D8Mg1LNRZILZ6XG8I5dJ4KuAd9Q==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/run-script": { + "version": "10.0.4", + "resolved": "https://registry.npmjs.org/@npmcli/run-script/-/run-script-10.0.4.tgz", + "integrity": "sha512-mGUWr1uMnf0le2TwfOZY4SFxZGXGfm4Jtay/nwAa2FLNAKXUoUwaGwBMNH36UHPtinWfTSJ3nqFQr0091CxVGg==", + "license": "ISC", + "dependencies": { + "@npmcli/node-gyp": "^5.0.0", + "@npmcli/package-json": "^7.0.0", + "@npmcli/promise-spawn": "^9.0.0", + "node-gyp": "^12.1.0", + "proc-log": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/run-script/node_modules/@npmcli/git": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@npmcli/git/-/git-7.0.2.tgz", + "integrity": "sha512-oeolHDjExNAJAnlYP2qzNjMX/Xi9bmu78C9dIGr4xjobrSKbuMYCph8lTzn4vnW3NjIqVmw/f8BCfouqyJXlRg==", + "license": "ISC", + "dependencies": { + "@gar/promise-retry": "^1.0.0", + "@npmcli/promise-spawn": "^9.0.0", + "ini": "^6.0.0", + "lru-cache": "^11.2.1", + "npm-pick-manifest": "^11.0.1", + "proc-log": "^6.0.0", + "semver": "^7.3.5", + "which": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/run-script/node_modules/@npmcli/package-json": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@npmcli/package-json/-/package-json-7.0.5.tgz", + "integrity": "sha512-iVuTlG3ORq2iaVa1IWUxAO/jIp77tUKBhoMjuzYW2kL4MLN1bi/ofqkZ7D7OOwh8coAx1/S2ge0rMdGv8sLSOQ==", + "license": "ISC", + "dependencies": { + "@npmcli/git": "^7.0.0", + "glob": "^13.0.0", + "hosted-git-info": "^9.0.0", + "json-parse-even-better-errors": "^5.0.0", + "proc-log": "^6.0.0", + "semver": "^7.5.3", + "spdx-expression-parse": "^4.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/run-script/node_modules/@npmcli/promise-spawn": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-9.0.1.tgz", + "integrity": "sha512-OLUaoqBuyxeTqUvjA3FZFiXUfYC1alp3Sa99gW3EUDz3tZ3CbXDdcZ7qWKBzicrJleIgucoWamWH1saAmH/l2Q==", + "license": "ISC", + "dependencies": { + "which": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/run-script/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@npmcli/run-script/node_modules/hosted-git-info": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.2.tgz", + "integrity": "sha512-M422h7o/BR3rmCQ8UHi7cyyMqKltdP9Uo+J2fXK+RSAY+wTcKOIRyhTuKv4qn+DJf3g+PL890AzId5KZpX+CBg==", + "license": "ISC", + "dependencies": { + "lru-cache": "^11.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/run-script/node_modules/ini": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-6.0.0.tgz", + "integrity": "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/run-script/node_modules/isexe": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", + "integrity": "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=20" + } + }, + "node_modules/@npmcli/run-script/node_modules/json-parse-even-better-errors": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-5.0.0.tgz", + "integrity": "sha512-ZF1nxZ28VhQouRWhUcVlUIN3qwSgPuswK05s/HIaoetAoE/9tngVmCHjSxmSQPav1nd+lPtTL0YZ/2AFdR/iYQ==", + "license": "MIT", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/run-script/node_modules/lru-cache": { + "version": "11.2.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", + "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@npmcli/run-script/node_modules/npm-pick-manifest": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-11.0.3.tgz", + "integrity": "sha512-buzyCfeoGY/PxKqmBqn1IUJrZnUi1VVJTdSSRPGI60tJdUhUoSQFhs0zycJokDdOznQentgrpf8LayEHyyYlqQ==", + "license": "ISC", + "dependencies": { + "npm-install-checks": "^8.0.0", + "npm-normalize-package-bin": "^5.0.0", + "npm-package-arg": "^13.0.0", + "semver": "^7.3.5" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/run-script/node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@npmcli/run-script/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@npmcli/run-script/node_modules/spdx-expression-parse": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-4.0.0.tgz", + "integrity": "sha512-Clya5JIij/7C6bRR22+tnGXbc4VKlibKSVj2iHvVeX5iMW7s1SIQlqu699JkODJJIhh/pUu8L0/VLh8xflD+LQ==", + "license": "MIT", + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/@npmcli/run-script/node_modules/which": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", + "integrity": "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==", + "license": "ISC", + "dependencies": { + "isexe": "^4.0.0" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.55.0.tgz", + "integrity": "sha512-3cpa+qI45VHYcA5c0bHM6VHo9gicv3p5mlLHNG3rLyjQU8b7e0st1rWtrUn3JbZ3DwwCfhKop4eQ9UuYlC6Pkg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.30.1.tgz", + "integrity": "sha512-s5vvxXPVdjqS3kTLKMeBMvop9hbWkwzBpu+mUO2M7sZtlkyDJGwFe33wRKnbaYDo8ExRVBIIdwIGrqpxHuKttA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.28.0.tgz", + "integrity": "sha512-ZLwRMV+fNDpVmF2WYUdBHlq0eOWtEaUJSusrzjGnBt7iSRvfjFE3RXYUZJrqou/wIDWV0DwQ5KIfYe9WXg9Xqw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.55.0.tgz", + "integrity": "sha512-ohIkCLn2Wc3vhhFuf1bH8kOXHMEdcWiD847x7f3Qfygc+CGiatGLzQYscTcEYsWGMV22gVwB/kVcNcx5a3o8gA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-trace-base": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/resources": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.28.0.tgz", + "integrity": "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-zipkin": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-zipkin/-/exporter-zipkin-1.30.1.tgz", + "integrity": "sha512-6S2QIMJahIquvFaaxmcwpvQQRD/YFaMTNoIxrfPIPOeITN+a8lfEcPDxNxn8JDAaxkg+4EnXhz8upVDYenoQjA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/exporter-zipkin/node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-zipkin/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/instrumentation": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.55.0.tgz", + "integrity": "sha512-YDCMlaQRZkziLL3t6TONRgmmGxDx6MyQDXRD0dknkkgUZtOK5+8MWft1OXzmNu6XfBOdT12MKN5rz+jHUkafKQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.55.0", + "@types/shimmer": "^1.2.0", + "import-in-the-middle": "^1.8.1", + "require-in-the-middle": "^7.1.1", + "semver": "^7.5.2", + "shimmer": "^1.2.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/instrumentation-grpc": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-grpc/-/instrumentation-grpc-0.55.0.tgz", + "integrity": "sha512-n2ZH4pRwOy0Vhag/3eKqiyDBwcpUnGgJI9iiIRX7vivE0FMncaLazWphNFezRRaM/LuKwq1TD8pVUvieP68mow==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/instrumentation": "0.55.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.55.0.tgz", + "integrity": "sha512-iHQI0Zzq3h1T6xUJTVFwmFl5Dt5y1es+fl4kM+k5T/3YvmVyeYkSiF+wHCg6oKrlUAJfk+t55kaAu3sYmt7ZYA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-transformer": "0.55.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-grpc-exporter-base": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.55.0.tgz", + "integrity": "sha512-gebbjl9FiSp52igWXuGjcWQKfB6IBwFGt5z1VFwTcVZVeEZevB6bJIqoFrhH4A02m7OUlpJ7l4EfRi3UtkNANQ==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/otlp-exporter-base": "0.55.0", + "@opentelemetry/otlp-transformer": "0.55.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.55.0.tgz", + "integrity": "sha512-kVqEfxtp6mSN2Dhpy0REo1ghP4PYhC1kMHQJ2qVlO99Pc+aigELjZDfg7/YKmL71gR6wVGIeJfiql/eXL7sQPA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.55.0", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/sdk-logs": "0.55.0", + "@opentelemetry/sdk-metrics": "1.28.0", + "@opentelemetry/sdk-trace-base": "1.28.0", + "protobufjs": "^7.3.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/resources": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.28.0.tgz", + "integrity": "sha512-ceUVWuCpIao7Y5xE02Xs3nQi0tOGmMea17ecBdwtCvdo9ekmO+ijc9RFDgfifMl7XCBf41zne/1POM3LqSTZDA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-b3": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-b3/-/propagator-b3-1.30.1.tgz", + "integrity": "sha512-oATwWWDIJzybAZ4pO76ATN5N6FFbOA1otibAVlS8v90B4S1wClnhRUk7K+2CHAwN1JKYuj4jh/lpCEG5BAqFuQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-b3/node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-b3/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/propagator-jaeger": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.30.1.tgz", + "integrity": "sha512-Pj/BfnYEKIOImirH76M4hDaBSx6HyZ2CXUqk+Kj02m6BB80c/yo4BdWkn/1gDFfU+YPY+bPR2U0DKBfdxCKwmg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-jaeger/node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-jaeger/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.30.1.tgz", + "integrity": "sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.55.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.55.0.tgz", + "integrity": "sha512-TSx+Yg/d48uWW6HtjS1AD5x6WPfLhDWLl/WxC7I2fMevaiBuKCuraxTB8MDXieCNnBI24bw9ytyXrDCswFfWgA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.55.0", + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/resources": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.28.0.tgz", + "integrity": "sha512-43tqMK/0BcKTyOvm15/WQ3HLr0Vu/ucAl/D84NO7iSlv6O4eOprxSHa3sUtmYkaZWHqdDJV0AHVz/R6u4JALVQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/resources": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/resources": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.28.0.tgz", + "integrity": "sha512-cIyXSVJjGeTICENN40YSvLDAq4Y2502hGK3iN7tfdynQLKWb3XWZQEkPc+eSx47kiy11YeFAlYkEfXwR1w8kfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.28.0", + "@opentelemetry/semantic-conventions": "1.27.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.30.1.tgz", + "integrity": "sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/resources": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.30.1.tgz", + "integrity": "sha512-cBjYOINt1JxXdpw1e5MlHmFRc5fgj4GW/86vsKFxJCJ8AL4PdVtYH41gWwl4qd4uQjqEL1oJVrXkSy5cnduAnQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "1.30.1", + "@opentelemetry/core": "1.30.1", + "@opentelemetry/propagator-b3": "1.30.1", + "@opentelemetry/propagator-jaeger": "1.30.1", + "@opentelemetry/sdk-trace-base": "1.30.1", + "semver": "^7.5.2" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node/node_modules/@opentelemetry/core": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", + "integrity": "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node/node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.27.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz", + "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, + "node_modules/@pulumi/command": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@pulumi/command/-/command-1.2.1.tgz", + "integrity": "sha512-mutNDIUYP67yCBYOVIidQyxuTwZDY9v/sx9EGbgIv4PXfyfolOKGgGLeoHEbI1lxRwaw2wbTZ3VNIynDnA5VKA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@pulumi/pulumi": "^3.142.0" + } + }, + "node_modules/@pulumi/gcp": { + "version": "9.14.0", + "resolved": "https://registry.npmjs.org/@pulumi/gcp/-/gcp-9.14.0.tgz", + "integrity": "sha512-5QZelNAR1VkvbP9AFPCJUv2jQTgTAe0B/ed197ZucOpuL0HR/m27ggElW3RxE96qP5GXtd9KArU5SFcoVZ05yA==", + "license": "Apache-2.0", + "dependencies": { + "@npmcli/package-json": "^6.2.0", + "@pulumi/pulumi": "^3.142.0", + "@types/express": "^4.16.0" + } + }, + "node_modules/@pulumi/pulumi": { + "version": "3.225.0", + "resolved": "https://registry.npmjs.org/@pulumi/pulumi/-/pulumi-3.225.0.tgz", + "integrity": "sha512-dqlc+d7kd6srAEyLxhO/lHRj0AWSvaMYNbP2BWafXZuzqp/2zg0Ro+OPE2/dQbyJQwW3bD250DLzEU94qInlcw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.10.1", + "@logdna/tail-file": "^2.0.6", + "@npmcli/arborist": "^9.0.0", + "@opentelemetry/api": "^1.9", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.55", + "@opentelemetry/exporter-zipkin": "^1.28", + "@opentelemetry/instrumentation": "^0.55", + "@opentelemetry/instrumentation-grpc": "^0.55", + "@opentelemetry/resources": "^1.28", + "@opentelemetry/sdk-trace-base": "^1.28", + "@opentelemetry/sdk-trace-node": "^1.28", + "@types/google-protobuf": "^3.15.5", + "@types/semver": "^7.5.6", + "@types/tmp": "^0.2.6", + "execa": "^5.1.0", + "fdir": "^6.5.0", + "google-protobuf": "^3.21.4", + "got": "^11.8.6", + "ini": "^2.0.0", + "js-yaml": "^3.14.2", + "minimist": "^1.2.6", + "normalize-package-data": "^6.0.0", + "package-directory": "^8.1.0", + "picomatch": "^3.0.1", + "require-from-string": "^2.0.1", + "semver": "^7.5.2", + "source-map-support": "^0.5.6", + "tmp": "^0.2.4", + "upath": "^1.1.0" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "ts-node": ">= 7.0.1 < 12", + "typescript": ">= 3.8.3 < 6" + }, + "peerDependenciesMeta": { + "ts-node": { + "optional": true + }, + "typescript": { + "optional": true + } + } + }, + "node_modules/@pulumi/pulumi/node_modules/ini": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz", + "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/@sigstore/bundle": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@sigstore/bundle/-/bundle-4.0.0.tgz", + "integrity": "sha512-NwCl5Y0V6Di0NexvkTqdoVfmjTaQwoLM236r89KEojGmq/jMls8S+zb7yOwAPdXvbwfKDlP+lmXgAL4vKSQT+A==", + "license": "Apache-2.0", + "dependencies": { + "@sigstore/protobuf-specs": "^0.5.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@sigstore/core": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@sigstore/core/-/core-3.1.0.tgz", + "integrity": "sha512-o5cw1QYhNQ9IroioJxpzexmPjfCe7gzafd2RY3qnMpxr4ZEja+Jad/U8sgFpaue6bOaF+z7RVkyKVV44FN+N8A==", + "license": "Apache-2.0", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@sigstore/protobuf-specs": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@sigstore/protobuf-specs/-/protobuf-specs-0.5.0.tgz", + "integrity": "sha512-MM8XIwUjN2bwvCg1QvrMtbBmpcSHrkhFSCu1D11NyPvDQ25HEc4oG5/OcQfd/Tlf/OxmKWERDj0zGE23jQaMwA==", + "license": "Apache-2.0", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/@sigstore/sign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@sigstore/sign/-/sign-4.1.0.tgz", + "integrity": "sha512-Vx1RmLxLGnSUqx/o5/VsCjkuN5L7y+vxEEwawvc7u+6WtX2W4GNa7b9HEjmcRWohw/d6BpATXmvOwc78m+Swdg==", + "license": "Apache-2.0", + "dependencies": { + "@sigstore/bundle": "^4.0.0", + "@sigstore/core": "^3.1.0", + "@sigstore/protobuf-specs": "^0.5.0", + "make-fetch-happen": "^15.0.3", + "proc-log": "^6.1.0", + "promise-retry": "^2.0.1" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@sigstore/sign/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@sigstore/tuf": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@sigstore/tuf/-/tuf-4.0.1.tgz", + "integrity": "sha512-OPZBg8y5Vc9yZjmWCHrlWPMBqW5yd8+wFNl+thMdtcWz3vjVSoJQutF8YkrzI0SLGnkuFof4HSsWUhXrf219Lw==", + "license": "Apache-2.0", + "dependencies": { + "@sigstore/protobuf-specs": "^0.5.0", + "tuf-js": "^4.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@sigstore/verify": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@sigstore/verify/-/verify-3.1.0.tgz", + "integrity": "sha512-mNe0Iigql08YupSOGv197YdHpPPr+EzDZmfCgMc7RPNaZTw5aLN01nBl6CHJOh3BGtnMIj83EeN4butBchc8Ag==", + "license": "Apache-2.0", + "dependencies": { + "@sigstore/bundle": "^4.0.0", + "@sigstore/core": "^3.1.0", + "@sigstore/protobuf-specs": "^0.5.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@sindresorhus/is": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", + "integrity": "sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/is?sponsor=1" + } + }, + "node_modules/@szmarczak/http-timer": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz", + "integrity": "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==", + "license": "MIT", + "dependencies": { + "defer-to-connect": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@tufjs/canonical-json": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tufjs/canonical-json/-/canonical-json-2.0.0.tgz", + "integrity": "sha512-yVtV8zsdo8qFHe+/3kw81dSLyF7D576A5cCFCi4X7B39tWT7SekaEFUnvnWJHz+9qO7qJTah1JbrDjWKqFtdWA==", + "license": "MIT", + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/@tufjs/models": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@tufjs/models/-/models-4.1.0.tgz", + "integrity": "sha512-Y8cK9aggNRsqJVaKUlEYs4s7CvQ1b1ta2DVPyAimb0I2qhzjNk+A+mxvll/klL0RlfuIUei8BF7YWiua4kQqww==", + "license": "MIT", + "dependencies": { + "@tufjs/canonical-json": "2.0.0", + "minimatch": "^10.1.1" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.6", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", + "integrity": "sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==", + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/cacheable-request": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", + "integrity": "sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==", + "license": "MIT", + "dependencies": { + "@types/http-cache-semantics": "*", + "@types/keyv": "^3.1.4", + "@types/node": "*", + "@types/responselike": "^1.0.0" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/express": { + "version": "4.17.25", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", + "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==", + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "^1" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.19.8", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz", + "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/google-protobuf": { + "version": "3.15.12", + "resolved": "https://registry.npmjs.org/@types/google-protobuf/-/google-protobuf-3.15.12.tgz", + "integrity": "sha512-40um9QqwHjRS92qnOaDpL7RmDK15NuZYo9HihiJRbYkMQZlWnuH8AdvbMy8/o6lgLmKbDUKa+OALCltHdbOTpQ==", + "license": "MIT" + }, + "node_modules/@types/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==", + "license": "MIT" + }, + "node_modules/@types/http-errors": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.5.tgz", + "integrity": "sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==", + "license": "MIT" + }, + "node_modules/@types/keyv": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.4.tgz", + "integrity": "sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "25.3.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.3.3.tgz", + "integrity": "sha512-DpzbrH7wIcBaJibpKo9nnSQL0MTRdnWttGyE5haGwK86xgMOkFLp7vEyfQPGLOJh5wNYiJ3V9PmUMDhV9u8kkQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/@types/qs": { + "version": "6.14.0", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.14.0.tgz", + "integrity": "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==", + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "license": "MIT" + }, + "node_modules/@types/responselike": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.3.tgz", + "integrity": "sha512-H/+L+UkTV33uf49PH5pCAUBVPNj2nDBXTN+qS1dOwyyg24l3CcicicCA7ca+HMvJBZcFgl5r8e+RR6elsb4Lyw==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/semver": { + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-FmgJfu+MOcQ370SD0ev7EI8TlCAfKYU+B4m5T3yXc1CiRN94g/SZPtsCkk506aUDtlMnFZvasDwHHUcZUEaYuA==", + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@types/send/-/send-1.2.1.tgz", + "integrity": "sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.10", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz", + "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==", + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "<1" + } + }, + "node_modules/@types/serve-static/node_modules/@types/send": { + "version": "0.17.6", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz", + "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==", + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/shimmer": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@types/shimmer/-/shimmer-1.2.0.tgz", + "integrity": "sha512-UE7oxhQLLd9gub6JKIAhDq06T0F6FnztwMNRvYgjeQSBeMc1ZG/tA47EwfduvkuQS8apbkM/lpLpWsaCeYsXVg==", + "license": "MIT" + }, + "node_modules/@types/tmp": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/@types/tmp/-/tmp-0.2.6.tgz", + "integrity": "sha512-chhaNf2oKHlRkDGt+tiKE2Z5aJ6qalm7Z9rlLdBwmOiAAf09YQvvoLXjWK4HWPF1xU/fqvMgfNfpVoBscA/tKA==", + "license": "MIT" + }, + "node_modules/abbrev": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-4.0.0.tgz", + "integrity": "sha512-a1wflyaL0tHtJSmLSOVybYhy22vRih4eduhhrkcjgrWGnRfrZtovJ2FRjxuTtkkj47O/baf0R86QU5OuYpz8fA==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/acorn": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", + "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^8" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/bin-links": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/bin-links/-/bin-links-6.0.0.tgz", + "integrity": "sha512-X4CiKlcV2GjnCMwnKAfbVWpHa++65th9TuzAEYtZoATiOE2DQKhSp4CJlyLoTqdhBKlXjpXjCTYPNNFS33Fi6w==", + "license": "ISC", + "dependencies": { + "cmd-shim": "^8.0.0", + "npm-normalize-package-bin": "^5.0.0", + "proc-log": "^6.0.0", + "read-cmd-shim": "^6.0.0", + "write-file-atomic": "^7.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/bin-links/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", + "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/cacache": { + "version": "20.0.3", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-20.0.3.tgz", + "integrity": "sha512-3pUp4e8hv07k1QlijZu6Kn7c9+ZpWWk4j3F8N3xPuCExULobqJydKYOTj1FTq58srkJsXvO7LbGAH4C0ZU3WGw==", + "license": "ISC", + "dependencies": { + "@npmcli/fs": "^5.0.0", + "fs-minipass": "^3.0.0", + "glob": "^13.0.0", + "lru-cache": "^11.1.0", + "minipass": "^7.0.3", + "minipass-collect": "^2.0.1", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "p-map": "^7.0.2", + "ssri": "^13.0.0", + "unique-filename": "^5.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/cacache/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/cacache/node_modules/lru-cache": { + "version": "11.2.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", + "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/cacache/node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/cacheable-lookup": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", + "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==", + "license": "MIT", + "engines": { + "node": ">=10.6.0" + } + }, + "node_modules/cacheable-request": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.4.tgz", + "integrity": "sha512-v+p6ongsrp0yTGbJXjgxPow2+DL93DASP4kXCDKb8/bwRtt9OEF3whggkkDkGNzgcWy2XaF4a8nZglC7uElscg==", + "license": "MIT", + "dependencies": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^4.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^6.0.1", + "responselike": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cacheable-request/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", + "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", + "license": "MIT" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/clone-response": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.3.tgz", + "integrity": "sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA==", + "license": "MIT", + "dependencies": { + "mimic-response": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cmd-shim": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/cmd-shim/-/cmd-shim-8.0.0.tgz", + "integrity": "sha512-Jk/BK6NCapZ58BKUxlSI+ouKRbjH1NLZCgJkYoab+vEHUY3f6OzpNBN9u7HFSv9J6TRDGs4PLOHezoKGaFRSCA==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/common-ancestor-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/common-ancestor-path/-/common-ancestor-path-2.0.0.tgz", + "integrity": "sha512-dnN3ibLeoRf2HNC+OlCiNc5d2zxbLJXOtiZUudNFSXZrNSydxcCsSpRzXwfu7BBWCIfHPw+xTayeBvJCP/D8Ng==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">= 18" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/cross-spawn/node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/cross-spawn/node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "license": "MIT", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/decompress-response/node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/defer-to-connect": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", + "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/err-code": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", + "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", + "license": "MIT" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/exponential-backoff": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.3.tgz", + "integrity": "sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==", + "license": "Apache-2.0" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/find-up-simple": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/find-up-simple/-/find-up-simple-1.0.1.tgz", + "integrity": "sha512-afd4O7zpqHeRyg4PfDQsXmlDe2PfdHtJt6Akt8jOWaApLOZk5JXs6VMR29lz03pRe9mpykrRCYIYxaJYcfpncQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fs-minipass": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz", + "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob/node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/glob/node_modules/brace-expansion": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", + "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/glob/node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/google-protobuf": { + "version": "3.21.4", + "resolved": "https://registry.npmjs.org/google-protobuf/-/google-protobuf-3.21.4.tgz", + "integrity": "sha512-MnG7N936zcKTco4Jd2PX2U96Kf9PxygAPKBug+74LHzmHXmceN16MmRcdgZv+DGef/S9YvQAfRsNCn4cjf9yyQ==", + "license": "(BSD-3-Clause AND Apache-2.0)" + }, + "node_modules/got": { + "version": "11.8.6", + "resolved": "https://registry.npmjs.org/got/-/got-11.8.6.tgz", + "integrity": "sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g==", + "license": "MIT", + "dependencies": { + "@sindresorhus/is": "^4.0.0", + "@szmarczak/http-timer": "^4.0.5", + "@types/cacheable-request": "^6.0.1", + "@types/responselike": "^1.0.0", + "cacheable-lookup": "^5.0.3", + "cacheable-request": "^7.0.2", + "decompress-response": "^6.0.0", + "http2-wrapper": "^1.0.0-beta.5.2", + "lowercase-keys": "^2.0.0", + "p-cancelable": "^2.0.0", + "responselike": "^2.0.0" + }, + "engines": { + "node": ">=10.19.0" + }, + "funding": { + "url": "https://github.com/sindresorhus/got?sponsor=1" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hosted-git-info": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-8.1.0.tgz", + "integrity": "sha512-Rw/B2DNQaPBICNXEm8balFz9a6WpZrkCGpcWFpy7nCj+NyhSdqXipmfvtmWt9xGfp0wZnBxB+iVpLmQMYt47Tw==", + "license": "ISC", + "dependencies": { + "lru-cache": "^10.0.1" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", + "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", + "license": "BSD-2-Clause" + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/http2-wrapper": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz", + "integrity": "sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==", + "license": "MIT", + "dependencies": { + "quick-lru": "^5.1.1", + "resolve-alpn": "^1.0.0" + }, + "engines": { + "node": ">=10.19.0" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ignore-walk": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-8.0.0.tgz", + "integrity": "sha512-FCeMZT4NiRQGh+YkeKMtWrOmBgWjHjMJ26WQWrRQyoyzqevdaGSakUaJW5xQYmjLlUVk2qUnCjYVBax9EKKg8A==", + "license": "ISC", + "dependencies": { + "minimatch": "^10.0.3" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/import-in-the-middle": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-1.15.0.tgz", + "integrity": "sha512-bpQy+CrsRmYmoPMAE/0G33iwRqwW4ouqdRg8jgbH3aKuCtOc8lxgmYXg2dMM92CRiGP660EtBcymH/eVUpCSaA==", + "license": "Apache-2.0", + "dependencies": { + "acorn": "^8.14.0", + "acorn-import-attributes": "^1.9.5", + "cjs-module-lexer": "^1.2.2", + "module-details-from-path": "^1.0.3" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/ini": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-5.0.0.tgz", + "integrity": "sha512-+N0ngpO3e7cRUWOJAS7qw0IZIVc6XPrW4MlFBdD066F2L4k1L6ker3hLqSq7iXxU5tgS4WGkIUElWn5vogAEnw==", + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/ip-address": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz", + "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-3.1.5.tgz", + "integrity": "sha512-6B3tLtFqtQS4ekarvLVMZ+X+VlvQekbe4taUkf/rhVO3d/h0M2rfARm/pXLcPEsjjMsFgrFgSrhQIxcSVrBz8w==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/js-yaml": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "license": "MIT" + }, + "node_modules/json-parse-even-better-errors": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-4.0.0.tgz", + "integrity": "sha512-lR4MXjGNgkJc7tkQ97kb2nuEMnNCyU//XYVH0MKTGcXEiSudQ5MKGKen3C5QubYy0vmq+JGitUg92uuywGEwIA==", + "license": "MIT", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/json-stringify-nice": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/json-stringify-nice/-/json-stringify-nice-1.1.4.tgz", + "integrity": "sha512-5Z5RFW63yxReJ7vANgW6eZFGWaQvnPE3WNmZoOJrSkGju2etKA2L5rrOa1sm877TVTFt57A80BH1bArcmlLfPw==", + "license": "ISC", + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", + "engines": [ + "node >= 0.2.0" + ], + "license": "MIT" + }, + "node_modules/just-diff": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/just-diff/-/just-diff-6.0.2.tgz", + "integrity": "sha512-S59eriX5u3/QhMNq3v/gm8Kd0w8OS6Tz2FS1NG4blv+z0MuQcBRJyFWjdovM0Rad4/P4aUPFtnkNjMjyMlMSYA==", + "license": "MIT" + }, + "node_modules/just-diff-apply": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/just-diff-apply/-/just-diff-apply-5.5.0.tgz", + "integrity": "sha512-OYTthRfSh55WOItVqwpefPtNt2VdKsq5AnAK6apdtR6yCH8pr0CmSr710J0Mf+WdQy7K/OzMy7K2MgAfdQURDw==", + "license": "MIT" + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "license": "ISC" + }, + "node_modules/make-fetch-happen": { + "version": "15.0.4", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-15.0.4.tgz", + "integrity": "sha512-vM2sG+wbVeVGYcCm16mM3d5fuem9oC28n436HjsGO3LcxoTI8LNVa4rwZDn3f76+cWyT4GGJDxjTYU1I2nr6zw==", + "license": "ISC", + "dependencies": { + "@gar/promise-retry": "^1.0.0", + "@npmcli/agent": "^4.0.0", + "cacache": "^20.0.1", + "http-cache-semantics": "^4.1.1", + "minipass": "^7.0.2", + "minipass-fetch": "^5.0.0", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^1.0.0", + "proc-log": "^6.0.0", + "ssri": "^13.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/make-fetch-happen/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "license": "MIT" + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/minimatch": { + "version": "10.2.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", + "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minipass-collect": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-2.0.1.tgz", + "integrity": "sha512-D7V8PO9oaz7PWGLbCACuI1qEOsq7UKfLotx/C0Aet43fCUB/wfQ7DYeq2oR/svFJGYDHPr38SHATeaj/ZoKHKw==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minipass-fetch": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-5.0.2.tgz", + "integrity": "sha512-2d0q2a8eCi2IRg/IGubCNRJoYbA1+YPXAzQVRFmB45gdGZafyivnZ5YSEfo3JikbjGxOdntGFvBQGqaSMXlAFQ==", + "license": "MIT", + "dependencies": { + "minipass": "^7.0.3", + "minipass-sized": "^2.0.0", + "minizlib": "^3.0.1" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + }, + "optionalDependencies": { + "iconv-lite": "^0.7.2" + } + }, + "node_modules/minipass-flush": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", + "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-flush/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-flush/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "license": "ISC", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, + "node_modules/minipass-sized": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-2.0.0.tgz", + "integrity": "sha512-zSsHhto5BcUVM2m1LurnXY6M//cGhVaegT71OfOXoprxT6o780GZd792ea6FfrQkuU4usHZIUczAQMRUE2plzA==", + "license": "ISC", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-gyp": { + "version": "12.2.0", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-12.2.0.tgz", + "integrity": "sha512-q23WdzrQv48KozXlr0U1v9dwO/k59NHeSzn6loGcasyf0UnSrtzs8kRxM+mfwJSf0DkX0s43hcqgnSO4/VNthQ==", + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.0", + "exponential-backoff": "^3.1.1", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^15.0.0", + "nopt": "^9.0.0", + "proc-log": "^6.0.0", + "semver": "^7.3.5", + "tar": "^7.5.4", + "tinyglobby": "^0.2.12", + "which": "^6.0.0" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/node-gyp/node_modules/isexe": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", + "integrity": "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=20" + } + }, + "node_modules/node-gyp/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/node-gyp/node_modules/which": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", + "integrity": "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==", + "license": "ISC", + "dependencies": { + "isexe": "^4.0.0" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/nopt": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-9.0.0.tgz", + "integrity": "sha512-Zhq3a+yFKrYwSBluL4H9XP3m3y5uvQkB/09CwDruCiRmR/UJYnn9W4R48ry0uGC70aeTPKLynBtscP9efFFcPw==", + "license": "ISC", + "dependencies": { + "abbrev": "^4.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/normalize-package-data": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-6.0.2.tgz", + "integrity": "sha512-V6gygoYb/5EmNI+MEGrWkC+e6+Rr7mTmfHrxDbLzxQogBkgzo76rkok0Am6thgSF7Mv2nLOajAJj5vDJZEFn7g==", + "license": "BSD-2-Clause", + "dependencies": { + "hosted-git-info": "^7.0.0", + "semver": "^7.3.5", + "validate-npm-package-license": "^3.0.4" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/normalize-package-data/node_modules/hosted-git-info": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-7.0.2.tgz", + "integrity": "sha512-puUZAUKT5m8Zzvs72XWy3HtvVbTWljRE66cP60bxJzAqf2DgICo7lYTY2IHUmLnNpjYvw5bvmoHvPc0QO2a62w==", + "license": "ISC", + "dependencies": { + "lru-cache": "^10.0.1" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/normalize-url": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", + "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-bundled": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-5.0.0.tgz", + "integrity": "sha512-JLSpbzh6UUXIEoqPsYBvVNVmyrjVZ1fzEFbqxKkTJQkWBO3xFzFT+KDnSKQWwOQNbuWRwt5LSD6HOTLGIWzfrw==", + "license": "ISC", + "dependencies": { + "npm-normalize-package-bin": "^5.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/npm-install-checks": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/npm-install-checks/-/npm-install-checks-8.0.0.tgz", + "integrity": "sha512-ScAUdMpyzkbpxoNekQ3tNRdFI8SJ86wgKZSQZdUxT+bj0wVFpsEMWnkXP0twVe1gJyNF5apBWDJhhIbgrIViRA==", + "license": "BSD-2-Clause", + "dependencies": { + "semver": "^7.1.1" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/npm-normalize-package-bin": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-5.0.0.tgz", + "integrity": "sha512-CJi3OS4JLsNMmr2u07OJlhcrPxCeOeP/4xq67aWNai6TNWWbTrlNDgl8NcFKVlcBKp18GPj+EzbNIgrBfZhsag==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/npm-package-arg": { + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-13.0.2.tgz", + "integrity": "sha512-IciCE3SY3uE84Ld8WZU23gAPPV9rIYod4F+rc+vJ7h7cwAJt9Vk6TVsK60ry7Uj3SRS3bqRRIGuTp9YVlk6WNA==", + "license": "ISC", + "dependencies": { + "hosted-git-info": "^9.0.0", + "proc-log": "^6.0.0", + "semver": "^7.3.5", + "validate-npm-package-name": "^7.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/npm-package-arg/node_modules/hosted-git-info": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.2.tgz", + "integrity": "sha512-M422h7o/BR3rmCQ8UHi7cyyMqKltdP9Uo+J2fXK+RSAY+wTcKOIRyhTuKv4qn+DJf3g+PL890AzId5KZpX+CBg==", + "license": "ISC", + "dependencies": { + "lru-cache": "^11.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/npm-package-arg/node_modules/lru-cache": { + "version": "11.2.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", + "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/npm-package-arg/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/npm-packlist": { + "version": "10.0.4", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-10.0.4.tgz", + "integrity": "sha512-uMW73iajD8hiH4ZBxEV3HC+eTnppIqwakjOYuvgddnalIw2lJguKviK1pcUJDlIWm1wSJkchpDZDSVVsZEYRng==", + "license": "ISC", + "dependencies": { + "ignore-walk": "^8.0.0", + "proc-log": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/npm-packlist/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/npm-pick-manifest": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-10.0.0.tgz", + "integrity": "sha512-r4fFa4FqYY8xaM7fHecQ9Z2nE9hgNfJR+EmoKv0+chvzWkBcORX3r0FpTByP+CbOVJDladMXnPQGVN8PBLGuTQ==", + "license": "ISC", + "dependencies": { + "npm-install-checks": "^7.1.0", + "npm-normalize-package-bin": "^4.0.0", + "npm-package-arg": "^12.0.0", + "semver": "^7.3.5" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm-pick-manifest/node_modules/npm-install-checks": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/npm-install-checks/-/npm-install-checks-7.1.2.tgz", + "integrity": "sha512-z9HJBCYw9Zr8BqXcllKIs5nI+QggAImbBdHphOzVYrz2CB4iQ6FzWyKmlqDZua+51nAu7FcemlbTc9VgQN5XDQ==", + "license": "BSD-2-Clause", + "dependencies": { + "semver": "^7.1.1" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm-pick-manifest/node_modules/npm-normalize-package-bin": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-4.0.0.tgz", + "integrity": "sha512-TZKxPvItzai9kN9H/TkmCtx/ZN/hvr3vUycjlfmH0ootY9yFBzNOpiXAdIn1Iteqsvk4lQn6B5PTrt+n6h8k/w==", + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm-pick-manifest/node_modules/npm-package-arg": { + "version": "12.0.2", + "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-12.0.2.tgz", + "integrity": "sha512-f1NpFjNI9O4VbKMOlA5QoBq/vSQPORHcTZ2feJpFkTHJ9eQkdlmZEKSjcAhxTGInC7RlEyScT9ui67NaOsjFWA==", + "license": "ISC", + "dependencies": { + "hosted-git-info": "^8.0.0", + "proc-log": "^5.0.0", + "semver": "^7.3.5", + "validate-npm-package-name": "^6.0.0" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm-pick-manifest/node_modules/validate-npm-package-name": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-6.0.2.tgz", + "integrity": "sha512-IUoow1YUtvoBBC06dXs8bR8B9vuA3aJfmQNKMoaPG/OFsPmoQvw8xh+6Ye25Gx9DQhoEom3Pcu9MKHerm/NpUQ==", + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/npm-registry-fetch": { + "version": "19.1.1", + "resolved": "https://registry.npmjs.org/npm-registry-fetch/-/npm-registry-fetch-19.1.1.tgz", + "integrity": "sha512-TakBap6OM1w0H73VZVDf44iFXsOS3h+L4wVMXmbWOQroZgFhMch0juN6XSzBNlD965yIKvWg2dfu7NSiaYLxtw==", + "license": "ISC", + "dependencies": { + "@npmcli/redact": "^4.0.0", + "jsonparse": "^1.3.1", + "make-fetch-happen": "^15.0.0", + "minipass": "^7.0.2", + "minipass-fetch": "^5.0.0", + "minizlib": "^3.0.1", + "npm-package-arg": "^13.0.0", + "proc-log": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/npm-registry-fetch/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-cancelable": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", + "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/p-map": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.4.tgz", + "integrity": "sha512-tkAQEw8ysMzmkhgw8k+1U/iPhWNhykKnSk4Rd5zLoPJCuJaGRPo6YposrZgaxHKzDHdDWWZvE/Sk7hsL2X/CpQ==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-directory": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/package-directory/-/package-directory-8.2.0.tgz", + "integrity": "sha512-qJSu5Mo6tHmRxCy2KCYYKYgcfBdUpy9dwReaZD/xwf608AUk/MoRtIOWzgDtUeGeC7n/55yC3MI1Q+MbSoektw==", + "license": "MIT", + "dependencies": { + "find-up-simple": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, + "node_modules/pacote": { + "version": "21.4.0", + "resolved": "https://registry.npmjs.org/pacote/-/pacote-21.4.0.tgz", + "integrity": "sha512-DR7mn7HUOomAX1BORnpYy678qVIidbvOojkBscqy27dRKN+s/hLeQT1MeYYrx1Cxh62jyKjiWiDV7RTTqB+ZEQ==", + "license": "ISC", + "dependencies": { + "@gar/promise-retry": "^1.0.0", + "@npmcli/git": "^7.0.0", + "@npmcli/installed-package-contents": "^4.0.0", + "@npmcli/package-json": "^7.0.0", + "@npmcli/promise-spawn": "^9.0.0", + "@npmcli/run-script": "^10.0.0", + "cacache": "^20.0.0", + "fs-minipass": "^3.0.0", + "minipass": "^7.0.2", + "npm-package-arg": "^13.0.0", + "npm-packlist": "^10.0.1", + "npm-pick-manifest": "^11.0.1", + "npm-registry-fetch": "^19.0.0", + "proc-log": "^6.0.0", + "sigstore": "^4.0.0", + "ssri": "^13.0.0", + "tar": "^7.4.3" + }, + "bin": { + "pacote": "bin/index.js" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/pacote/node_modules/@npmcli/git": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@npmcli/git/-/git-7.0.2.tgz", + "integrity": "sha512-oeolHDjExNAJAnlYP2qzNjMX/Xi9bmu78C9dIGr4xjobrSKbuMYCph8lTzn4vnW3NjIqVmw/f8BCfouqyJXlRg==", + "license": "ISC", + "dependencies": { + "@gar/promise-retry": "^1.0.0", + "@npmcli/promise-spawn": "^9.0.0", + "ini": "^6.0.0", + "lru-cache": "^11.2.1", + "npm-pick-manifest": "^11.0.1", + "proc-log": "^6.0.0", + "semver": "^7.3.5", + "which": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/pacote/node_modules/@npmcli/package-json": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@npmcli/package-json/-/package-json-7.0.5.tgz", + "integrity": "sha512-iVuTlG3ORq2iaVa1IWUxAO/jIp77tUKBhoMjuzYW2kL4MLN1bi/ofqkZ7D7OOwh8coAx1/S2ge0rMdGv8sLSOQ==", + "license": "ISC", + "dependencies": { + "@npmcli/git": "^7.0.0", + "glob": "^13.0.0", + "hosted-git-info": "^9.0.0", + "json-parse-even-better-errors": "^5.0.0", + "proc-log": "^6.0.0", + "semver": "^7.5.3", + "spdx-expression-parse": "^4.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/pacote/node_modules/@npmcli/promise-spawn": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/@npmcli/promise-spawn/-/promise-spawn-9.0.1.tgz", + "integrity": "sha512-OLUaoqBuyxeTqUvjA3FZFiXUfYC1alp3Sa99gW3EUDz3tZ3CbXDdcZ7qWKBzicrJleIgucoWamWH1saAmH/l2Q==", + "license": "ISC", + "dependencies": { + "which": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/pacote/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/pacote/node_modules/hosted-git-info": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.2.tgz", + "integrity": "sha512-M422h7o/BR3rmCQ8UHi7cyyMqKltdP9Uo+J2fXK+RSAY+wTcKOIRyhTuKv4qn+DJf3g+PL890AzId5KZpX+CBg==", + "license": "ISC", + "dependencies": { + "lru-cache": "^11.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/pacote/node_modules/ini": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-6.0.0.tgz", + "integrity": "sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/pacote/node_modules/isexe": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", + "integrity": "sha512-FFUtZMpoZ8RqHS3XeXEmHWLA4thH+ZxCv2lOiPIn1Xc7CxrqhWzNSDzD+/chS/zbYezmiwWLdQC09JdQKmthOw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=20" + } + }, + "node_modules/pacote/node_modules/json-parse-even-better-errors": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-5.0.0.tgz", + "integrity": "sha512-ZF1nxZ28VhQouRWhUcVlUIN3qwSgPuswK05s/HIaoetAoE/9tngVmCHjSxmSQPav1nd+lPtTL0YZ/2AFdR/iYQ==", + "license": "MIT", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/pacote/node_modules/lru-cache": { + "version": "11.2.6", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", + "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/pacote/node_modules/npm-pick-manifest": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/npm-pick-manifest/-/npm-pick-manifest-11.0.3.tgz", + "integrity": "sha512-buzyCfeoGY/PxKqmBqn1IUJrZnUi1VVJTdSSRPGI60tJdUhUoSQFhs0zycJokDdOznQentgrpf8LayEHyyYlqQ==", + "license": "ISC", + "dependencies": { + "npm-install-checks": "^8.0.0", + "npm-normalize-package-bin": "^5.0.0", + "npm-package-arg": "^13.0.0", + "semver": "^7.3.5" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/pacote/node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/pacote/node_modules/proc-log": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-6.1.0.tgz", + "integrity": "sha512-iG+GYldRf2BQ0UDUAd6JQ/RwzaQy6mXmsk/IzlYyal4A4SNFw54MeH4/tLkF4I5WoWG9SQwuqWzS99jaFQHBuQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/pacote/node_modules/spdx-expression-parse": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-4.0.0.tgz", + "integrity": "sha512-Clya5JIij/7C6bRR22+tnGXbc4VKlibKSVj2iHvVeX5iMW7s1SIQlqu699JkODJJIhh/pUu8L0/VLh8xflD+LQ==", + "license": "MIT", + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/pacote/node_modules/which": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/which/-/which-6.0.1.tgz", + "integrity": "sha512-oGLe46MIrCRqX7ytPUf66EAYvdeMIZYn3WaocqqKZAxrBpkqHfL/qvTyJ/bTk5+AqHCjXmrv3CEWgy368zhRUg==", + "license": "ISC", + "dependencies": { + "isexe": "^4.0.0" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/parse-conflict-json": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/parse-conflict-json/-/parse-conflict-json-5.0.1.tgz", + "integrity": "sha512-ZHEmNKMq1wyJXNwLxyHnluPfRAFSIliBvbK/UiOceROt4Xh9Pz0fq49NytIaeaCUf5VR86hwQ/34FCcNU5/LKQ==", + "license": "ISC", + "dependencies": { + "json-parse-even-better-errors": "^5.0.0", + "just-diff": "^6.0.0", + "just-diff-apply": "^5.2.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/parse-conflict-json/node_modules/json-parse-even-better-errors": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-5.0.0.tgz", + "integrity": "sha512-ZF1nxZ28VhQouRWhUcVlUIN3qwSgPuswK05s/HIaoetAoE/9tngVmCHjSxmSQPav1nd+lPtTL0YZ/2AFdR/iYQ==", + "license": "MIT", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "license": "MIT" + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/picomatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-3.0.1.tgz", + "integrity": "sha512-I3EurrIQMlRc9IaAZnqRR044Phh2DXY+55o7uJ0V+hYZAcQYSuFWsc9q5PvyDHUSCe1Qxn/iBz+78s86zWnGag==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss-selector-parser": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-7.1.1.tgz", + "integrity": "sha512-orRsuYpJVw8LdAwqqLykBj9ecS5/cRHlI5+nvTo8LcCKmzDmqVORXtOIYEEQuL9D4BxtA1lm5isAqzQZCoQ6Eg==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/proc-log": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-5.0.0.tgz", + "integrity": "sha512-Azwzvl90HaF0aCz1JrDdXQykFakSSNPaPoiZ9fm5qJIMHioDZEi7OAdRwSm6rSoPtY3Qutnm3L7ogmg3dc+wbQ==", + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/proggy": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/proggy/-/proggy-4.0.0.tgz", + "integrity": "sha512-MbA4R+WQT76ZBm/5JUpV9yqcJt92175+Y0Bodg3HgiXzrmKu7Ggq+bpn6y6wHH+gN9NcyKn3yg1+d47VaKwNAQ==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/promise-all-reject-late": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/promise-all-reject-late/-/promise-all-reject-late-1.0.1.tgz", + "integrity": "sha512-vuf0Lf0lOxyQREH7GDIOUMLS7kz+gs8i6B+Yi8dC68a2sychGrHTJYghMBD6k7eUcH0H5P73EckCA48xijWqXw==", + "license": "ISC", + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/promise-call-limit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/promise-call-limit/-/promise-call-limit-3.0.2.tgz", + "integrity": "sha512-mRPQO2T1QQVw11E7+UdCJu7S61eJVWknzml9sC1heAdj1jxl0fWMBypIt9ZOcLFf8FkG995ZD7RnVk7HH72fZw==", + "license": "ISC", + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/promise-retry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", + "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "license": "MIT", + "dependencies": { + "err-code": "^2.0.2", + "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/promise-retry/node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/read-cmd-shim": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/read-cmd-shim/-/read-cmd-shim-6.0.0.tgz", + "integrity": "sha512-1zM5HuOfagXCBWMN83fuFI/x+T/UhZ7k+KIzhrHXcQoeX5+7gmaDYjELQHmmzIodumBHeByBJT4QYS7ufAgs7A==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-in-the-middle": { + "version": "7.5.2", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-7.5.2.tgz", + "integrity": "sha512-gAZ+kLqBdHarXB64XpAe2VCjB7rIRv+mU8tfRWziHRJ5umKsIHN2tLLv6EtMw7WCdP19S0ERVMldNvxYCHnhSQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "module-details-from-path": "^1.0.3", + "resolve": "^1.22.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/resolve": { + "version": "1.22.11", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", + "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-alpn": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", + "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==", + "license": "MIT" + }, + "node_modules/responselike": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.1.tgz", + "integrity": "sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw==", + "license": "MIT", + "dependencies": { + "lowercase-keys": "^2.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT", + "optional": true + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shimmer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==", + "license": "BSD-2-Clause" + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/sigstore": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/sigstore/-/sigstore-4.1.0.tgz", + "integrity": "sha512-/fUgUhYghuLzVT/gaJoeVehLCgZiUxPCPMcyVNY0lIf/cTCz58K/WTI7PefDarXxp9nUKpEwg1yyz3eSBMTtgA==", + "license": "Apache-2.0", + "dependencies": { + "@sigstore/bundle": "^4.0.0", + "@sigstore/core": "^3.1.0", + "@sigstore/protobuf-specs": "^0.5.0", + "@sigstore/sign": "^4.1.0", + "@sigstore/tuf": "^4.0.1", + "@sigstore/verify": "^3.1.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/spdx-correct": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", + "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", + "license": "Apache-2.0", + "dependencies": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-exceptions": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", + "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", + "license": "CC-BY-3.0" + }, + "node_modules/spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "license": "MIT", + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-license-ids": { + "version": "3.0.23", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.23.tgz", + "integrity": "sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==", + "license": "CC0-1.0" + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, + "node_modules/ssri": { + "version": "13.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-13.0.1.tgz", + "integrity": "sha512-QUiRf1+u9wPTL/76GTYlKttDEBWV1ga9ZXW8BG6kfdeyyM8LGPix9gROyg9V2+P0xNyF3X2Go526xKFdMZrHSQ==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/tar": { + "version": "7.5.10", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.10.tgz", + "integrity": "sha512-8mOPs1//5q/rlkNSPcCegA6hiHJYDmSLEI8aMH/CdSQJNWztHC9WHNam5zdQlfpTwB9Xp7IBEsHfV5LKMJGVAw==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.15", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", + "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/tmp": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", + "integrity": "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==", + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, + "node_modules/treeverse": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/treeverse/-/treeverse-3.0.0.tgz", + "integrity": "sha512-gcANaAnd2QDZFmHFEOF4k7uc1J/6a6z3DJMd/QwEyxLoKGiptJRwid582r7QIsFlFMIZ3SnxfS52S4hm2DHkuQ==", + "license": "ISC", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/tuf-js": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/tuf-js/-/tuf-js-4.1.0.tgz", + "integrity": "sha512-50QV99kCKH5P/Vs4E2Gzp7BopNV+KzTXqWeaxrfu5IQJBOULRsTIS9seSsOVT8ZnGXzCyx55nYWAi4qJzpZKEQ==", + "license": "MIT", + "dependencies": { + "@tufjs/models": "4.1.0", + "debug": "^4.4.3", + "make-fetch-happen": "^15.0.1" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + }, + "node_modules/unique-filename": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-5.0.0.tgz", + "integrity": "sha512-2RaJTAvAb4owyjllTfXzFClJ7WsGxlykkPvCr9pA//LD9goVq+m4PPAeBgNodGZ7nSrntT/auWpJ6Y5IFXcfjg==", + "license": "ISC", + "dependencies": { + "unique-slug": "^6.0.0" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/unique-slug": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-6.0.0.tgz", + "integrity": "sha512-4Lup7Ezn8W3d52/xBhZBVdx323ckxa7DEvd9kPQHppTkLoJXw6ltrBCyj5pnrxj0qKDxYMJ56CoxNuFCscdTiw==", + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/upath": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", + "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==", + "license": "MIT", + "engines": { + "node": ">=4", + "yarn": "*" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "license": "Apache-2.0", + "dependencies": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "node_modules/validate-npm-package-name": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/validate-npm-package-name/-/validate-npm-package-name-7.0.2.tgz", + "integrity": "sha512-hVDIBwsRruT73PbK7uP5ebUt+ezEtCmzZz3F59BSr2F6OVFnJ/6h8liuvdLrQ88Xmnk6/+xGGuq+pG9WwTuy3A==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/walk-up-path": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/walk-up-path/-/walk-up-path-4.0.0.tgz", + "integrity": "sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A==", + "license": "ISC", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/which": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/which/-/which-5.0.0.tgz", + "integrity": "sha512-JEdGzHwwkrbWoGOlIHqQ5gtprKGOenpDHpxE9zVR1bWbOtYRyPPHMe9FaP6x61CmNaTThSkb0DAJte5jD+DmzQ==", + "license": "ISC", + "dependencies": { + "isexe": "^3.1.1" + }, + "bin": { + "node-which": "bin/which.js" + }, + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/write-file-atomic": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-7.0.1.tgz", + "integrity": "sha512-OTIk8iR8/aCRWBqvxrzxR0hgxWpnYBblY1S5hDWBQfk/VFmJwzmJgQFN3WsoUKHISv2eAwe+PpbUzyL1CKTLXg==", + "license": "ISC", + "dependencies": { + "signal-exit": "^4.0.1" + }, + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, + "node_modules/write-file-atomic/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/ai-context/trustgraph-templates/pulumi/package.json b/ai-context/trustgraph-templates/pulumi/package.json new file mode 100644 index 00000000..1bbb56b1 --- /dev/null +++ b/ai-context/trustgraph-templates/pulumi/package.json @@ -0,0 +1,7 @@ +{ + "dependencies": { + "@pulumi/command": "^1.1.3", + "@pulumi/gcp": "^9.10.0", + "@pulumi/pulumi": "^3.216.0" + } +} diff --git a/ai-context/trustgraph-templates/pulumi/tsconfig.json b/ai-context/trustgraph-templates/pulumi/tsconfig.json new file mode 100644 index 00000000..ab65afa6 --- /dev/null +++ b/ai-context/trustgraph-templates/pulumi/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "strict": true, + "outDir": "bin", + "target": "es2016", + "module": "commonjs", + "moduleResolution": "node", + "sourceMap": true, + "experimentalDecorators": true, + "pretty": true, + "noFallthroughCasesInSwitch": true, + "noImplicitReturns": true, + "forceConsistentCasingInFileNames": true + }, + "files": [ + "index.ts" + ] +} diff --git a/ai-context/trustgraph-templates/pyproject.toml b/ai-context/trustgraph-templates/pyproject.toml new file mode 100644 index 00000000..953cd0a1 --- /dev/null +++ b/ai-context/trustgraph-templates/pyproject.toml @@ -0,0 +1,68 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "trustgraph-configurator" +dynamic = ["version"] +authors = [ + {name = "trustgraph.ai", email = "security@trustgraph.ai"}, +] +description = "Configuration creator for trustgraph.ai" +readme = "README.md" +requires-python = ">=3.8" +license = {text = "Apache-2.0"} +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +dependencies = [ + "aiohttp", + "gojsonnet", + "pyyaml", + "tabulate", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-xdist>=3.0", + "pytest-cov>=4.0", + "jsonschema>=4.0", +] + +[project.urls] +Homepage = "https://github.com/trustgraph-ai/trustgraph-configurator" + +[project.scripts] +tg-build-deployment = "trustgraph_configurator:run" +tg-config-svc = "trustgraph_configurator:run_service" +tg-show-config-params = "trustgraph_configurator:list" + +[tool.setuptools.packages.find] +include = ["trustgraph_configurator*"] + +[tool.setuptools.package-data] +trustgraph_configurator = ["templates/**", "resources/**"] + +[tool.setuptools.dynamic] +version = {attr = "trustgraph_configurator.__version__"} + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "-v", + "--strict-markers", + "--tb=short", +] +markers = [ + "unit: Unit tests", + "integration: Integration tests", + "validation: Output validation tests", + "slow: Slow-running tests", +] + diff --git a/ai-context/trustgraph-templates/test-dialog-flow.py b/ai-context/trustgraph-templates/test-dialog-flow.py new file mode 100644 index 00000000..a26d31ff --- /dev/null +++ b/ai-context/trustgraph-templates/test-dialog-flow.py @@ -0,0 +1,493 @@ +#!/usr/bin/env python3 +""" +Test harness for dialog flow - walks through selecting all default options, +produces the resulting state object, and runs the JSONata transform. + +Supports a test matrix mode where each field is tested with all its options +while other fields use defaults. +""" + +import yaml +import json +import argparse +import zipfile +import io +from pathlib import Path +import jsonata +import requests + + +RESOURCES_DIR = Path(__file__).parent / "trustgraph_configurator/resources/dialog" + + +def load_flow(): + """Load the dialog flow YAML file.""" + with open(RESOURCES_DIR / "trustgraph-flow.yaml") as f: + return yaml.safe_load(f) + + +def load_jsonata_transform(): + """Load the JSONata transform file.""" + with open(RESOURCES_DIR / "trustgraph-output.jsonata") as f: + return f.read() + + +def run_transform(state, transform_expr): + """Run the JSONata transform on the state object.""" + expr = jsonata.Jsonata(transform_expr) + return expr.evaluate(state) + + +def call_config_service(config): + """ + Call the configuration service to generate a deployment package. + Returns (success, message, zip_contents) tuple. + + The API expects just the templates array, not the full config object. + """ + url = config["api_url"] + templates = config["templates"] + + try: + response = requests.post( + url, + json=templates, + headers={"Content-Type": "application/json"}, + timeout=30 + ) + + if response.status_code != 200: + return False, f"HTTP {response.status_code}: {response.text[:200]}", None + + # Verify it's a valid ZIP + try: + zip_data = io.BytesIO(response.content) + with zipfile.ZipFile(zip_data, 'r') as zf: + file_list = zf.namelist() + return True, f"ZIP with {len(file_list)} files", file_list + except zipfile.BadZipFile: + return False, "Response is not a valid ZIP file", None + + except requests.exceptions.ConnectionError: + return False, "Connection refused - is the service running?", None + except requests.exceptions.Timeout: + return False, "Request timed out", None + except Exception as e: + return False, f"Error: {e}", None + + +def get_all_options(step): + """Get all possible values for a step's input.""" + input_def = step.get("input", {}) + input_type = input_def.get("type") + + if input_type == "select": + return [opt["value"] for opt in input_def.get("options", [])] + elif input_type == "toggle": + return [True, False] + elif input_type == "number": + # For numbers, just test default, min, and max + default = input_def.get("default") + min_val = input_def.get("min") + max_val = input_def.get("max") + values = [] + if min_val is not None: + values.append(min_val) + if default is not None and default not in values: + values.append(default) + if max_val is not None and max_val not in values: + values.append(max_val) + return values + + return [] + + +def get_default_value(step): + """Get the default value for a step's input.""" + input_def = step.get("input", {}) + input_type = input_def.get("type") + + if input_type == "select": + options = input_def.get("options", []) + # Find recommended option, or use first + for opt in options: + if opt.get("recommended"): + return opt["value"] + return options[0]["value"] if options else None + + elif input_type == "number": + return input_def.get("default") + + elif input_type == "toggle": + return input_def.get("default", False) + + return None + + +def evaluate_condition(condition, state): + """ + Evaluate a simple condition against the state. + Supports: "key = value", "key = true/false", "key < 'version'" + """ + if not condition: + return True + + # Handle equality: "ocr.enabled = true" + if " = " in condition: + key, value = condition.split(" = ", 1) + key = key.strip() + value = value.strip() + + # Get nested key + state_value = state.get(key) + + # Parse value + if value == "true": + return state_value is True + elif value == "false": + return state_value is False + else: + return str(state_value) == value + + # Handle less-than for version comparisons: "version < '1.6.0'" + if " < " in condition: + key, value = condition.split(" < ", 1) + key = key.strip() + value = value.strip().strip("'\"") + state_value = state.get(key, "") + return str(state_value) < value + + return False + + +def get_next_step(step, state): + """Determine the next step based on transitions and current state.""" + transitions = step.get("transitions", []) + + for trans in transitions: + when = trans.get("when") + if when: + if evaluate_condition(when, state): + return trans.get("next") + else: + # Unconditional transition + return trans.get("next") + + return None # Terminal state + + +def walk_flow(flow_data, overrides=None, verbose=True): + """ + Walk through the flow, return the state object. + + Args: + flow_data: The parsed dialog flow YAML + overrides: Dict of {state_key: value} to override defaults + verbose: Whether to print progress + """ + state = {} + overrides = overrides or {} + steps = flow_data.get("steps", {}) + current = flow_data.get("flow", {}).get("start") + visited_steps = [] + + if verbose: + print(f"Starting at: {current}") + print("-" * 60) + + while current: + step = steps.get(current) + if not step: + if verbose: + print(f"ERROR: Step '{current}' not found!") + break + + visited_steps.append(current) + title = step.get("title", current) + state_key = step.get("state_key") + + # Get value - use override if present, otherwise default + if state_key: + if state_key in overrides: + value = overrides[state_key] + else: + value = get_default_value(step) + state[state_key] = value + + if verbose: + is_override = state_key in overrides + marker = " [OVERRIDE]" if is_override else "" + print(f"Step: {current}") + print(f" Title: {title}") + print(f" State key: {state_key} = {value}{marker}") + else: + if verbose: + print(f"Step: {current}") + print(f" Title: {title}") + print(f" (no state key - review/terminal step)") + + # Get next step + next_step = get_next_step(step, state) + if verbose: + if next_step: + print(f" -> Next: {next_step}") + else: + print(f" -> Terminal state") + print() + + current = next_step + + return state, visited_steps + + +def collect_fields_for_path(flow_data, overrides=None): + """ + Collect all fields and their possible values for a given path. + Returns a list of (step_name, state_key, options, default_value) tuples. + """ + fields = [] + steps = flow_data.get("steps", {}) + + # Walk with given overrides to find the path + _, visited = walk_flow(flow_data, overrides=overrides, verbose=False) + + for step_name in visited: + step = steps.get(step_name, {}) + state_key = step.get("state_key") + if state_key: + options = get_all_options(step) + default = get_default_value(step) + if len(options) > 1: # Only include fields with choices + fields.append((step_name, state_key, options, default)) + + return fields + + +def collect_all_fields(flow_data): + """ + Collect all fields from the baseline (default) path. + """ + return collect_fields_for_path(flow_data, overrides=None) + + +def run_single_test(flow_data, transform_expr, overrides, description, test_num, results, call_api=False): + """Run a single test case and record the result.""" + print("-" * 70) + print(f"Test {test_num}: {description}") + print("-" * 70) + + state, _ = walk_flow(flow_data, overrides=overrides, verbose=False) + + try: + config = run_transform(state, transform_expr) + result = { + "test": test_num, + "description": description, + "overrides": overrides, + "state": state, + "config": config + } + + print(f"State: {json.dumps(state, indent=2)}") + print(f"Templates: {[t['name'] for t in config['templates']]}") + + # Optionally call the configuration service + if call_api: + success, message, files = call_config_service(config) + result["api_success"] = success + result["api_message"] = message + if files: + result["api_files"] = files + + if success: + print(f"API: OK - {message}") + else: + print(f"API: FAILED - {message}") + result["error"] = f"API: {message}" + + results.append(result) + + except Exception as e: + results.append({ + "test": test_num, + "description": description, + "overrides": overrides, + "state": state, + "error": str(e) + }) + print(f"State: {json.dumps(state, indent=2)}") + print(f"ERROR: {e}") + print() + + return test_num + 1 + + +def run_test_matrix(flow_data, transform_expr, call_api=False): + """ + Run the test matrix - for each field, try all values while others use defaults. + When a toggle enables conditional fields, also test all options of those fields. + """ + baseline_fields = collect_all_fields(flow_data) + baseline_keys = {f[1] for f in baseline_fields} + + print("=" * 70) + print("TEST MATRIX" + (" (with API validation)" if call_api else "")) + print("=" * 70) + print() + print(f"Found {len(baseline_fields)} fields with multiple options on baseline path:") + for step_name, state_key, options, default in baseline_fields: + print(f" - {state_key}: {len(options)} options (default: {default})") + print() + + results = [] + test_num = 1 + + # First, run the baseline (all defaults) + test_num = run_single_test( + flow_data, transform_expr, + overrides={}, + description="BASELINE (all defaults)", + test_num=test_num, + results=results, + call_api=call_api + ) + + # For each field, try each non-default value + for step_name, state_key, options, default in baseline_fields: + for option in options: + if option == default: + continue # Skip default, already tested in baseline + + overrides = {state_key: option} + test_num = run_single_test( + flow_data, transform_expr, + overrides=overrides, + description=f"{state_key} = {option}", + test_num=test_num, + results=results, + call_api=call_api + ) + + # Check if this override unlocks new fields (conditional paths) + unlocked_fields = collect_fields_for_path(flow_data, overrides=overrides) + unlocked_keys = {f[1] for f in unlocked_fields} + new_keys = unlocked_keys - baseline_keys + + if new_keys: + # Test all non-default options of the newly unlocked fields + for uf_step, uf_key, uf_options, uf_default in unlocked_fields: + if uf_key not in new_keys: + continue + for uf_option in uf_options: + if uf_option == uf_default: + continue # Default already tested above + + combined_overrides = {state_key: option, uf_key: uf_option} + test_num = run_single_test( + flow_data, transform_expr, + overrides=combined_overrides, + description=f"{state_key} = {option}, {uf_key} = {uf_option}", + test_num=test_num, + results=results, + call_api=call_api + ) + + return results + + +def main(): + parser = argparse.ArgumentParser( + description="Test harness for dialog flow configuration" + ) + parser.add_argument( + "--matrix", "-m", + action="store_true", + help="Run test matrix (each field with all options)" + ) + parser.add_argument( + "--api", "-a", + action="store_true", + help="Call the configuration service API to validate each config" + ) + parser.add_argument( + "--summary", "-s", + action="store_true", + help="Show summary only (with --matrix)" + ) + args = parser.parse_args() + + flow_data = load_flow() + transform_expr = load_jsonata_transform() + + if args.matrix: + results = run_test_matrix(flow_data, transform_expr, call_api=args.api) + + # Summary + print("=" * 70) + print("SUMMARY") + print("=" * 70) + print() + passed = [r for r in results if "error" not in r] + failed = [r for r in results if "error" in r] + print(f"Total tests: {len(results)}") + print(f"Passed: {len(passed)}") + print(f"Failed: {len(failed)}") + + if args.api: + api_ok = [r for r in results if r.get("api_success")] + api_fail = [r for r in results if "api_success" in r and not r["api_success"]] + print(f"API OK: {len(api_ok)}") + print(f"API Failed: {len(api_fail)}") + + if failed: + print() + print("Failed tests:") + for r in failed: + print(f" - Test {r['test']}: {r['description']}") + print(f" Error: {r['error']}") + else: + print("=" * 60) + print("Dialog Flow Test Harness - Default Options") + print("=" * 60) + print() + + state, _ = walk_flow(flow_data) + + print("=" * 60) + print("Final State Object:") + print("=" * 60) + print() + print(json.dumps(state, indent=2)) + + # Run JSONata transform + print() + print("=" * 60) + print("Running JSONata Transform...") + print("=" * 60) + print() + + config = run_transform(state, transform_expr) + + print("=" * 60) + print("Configuration Object (output of transform):") + print("=" * 60) + print() + print(json.dumps(config, indent=2)) + + # Optionally call the API + if args.api: + print() + print("=" * 60) + print("Calling Configuration Service...") + print("=" * 60) + print() + success, message, files = call_config_service(config) + if success: + print(f"OK: {message}") + print(f"Files: {files}") + else: + print(f"FAILED: {message}") + + +if __name__ == "__main__": + main() diff --git a/ai-context/trustgraph-templates/test-docs-flow.py b/ai-context/trustgraph-templates/test-docs-flow.py new file mode 100644 index 00000000..487a75c9 --- /dev/null +++ b/ai-context/trustgraph-templates/test-docs-flow.py @@ -0,0 +1,416 @@ +#!/usr/bin/env python3 +""" +Test harness for documentation assembly - verifies that for each configuration +state, the documentation can be assembled from the manifest and markdown fragments. +""" + +import yaml +import json +import argparse +import re +from pathlib import Path +import jsonata + +RESOURCES_DIR = Path(__file__).parent / "trustgraph_configurator/resources/dialog" + + +def load_flow(): + """Load the dialog flow YAML file.""" + with open(RESOURCES_DIR / "trustgraph-flow.yaml") as f: + return yaml.safe_load(f) + + +def load_docs_manifest(): + """Load the documentation manifest YAML file.""" + with open(RESOURCES_DIR / "trustgraph-docs.yaml") as f: + return yaml.safe_load(f) + + +def load_doc_file(filename): + """Load a documentation markdown file.""" + path = RESOURCES_DIR / "docs" / filename + if path.exists(): + return path.read_text() + return None + + +def get_default_value(step): + """Get the default value for a step's input.""" + input_def = step.get("input", {}) + input_type = input_def.get("type") + + if input_type == "select": + options = input_def.get("options", []) + for opt in options: + if opt.get("recommended"): + return opt["value"] + return options[0]["value"] if options else None + elif input_type == "number": + return input_def.get("default") + elif input_type == "toggle": + return input_def.get("default", False) + return None + + +def get_all_options(step): + """Get all possible values for a step's input.""" + input_def = step.get("input", {}) + input_type = input_def.get("type") + + if input_type == "select": + return [opt["value"] for opt in input_def.get("options", [])] + elif input_type == "toggle": + return [True, False] + elif input_type == "number": + default = input_def.get("default") + min_val = input_def.get("min") + max_val = input_def.get("max") + values = [] + if min_val is not None: + values.append(min_val) + if default is not None and default not in values: + values.append(default) + if max_val is not None and max_val not in values: + values.append(max_val) + return values + return [] + + +def evaluate_condition(condition, state): + """Evaluate a simple condition against the state.""" + if not condition: + return True + + if " = " in condition: + key, value = condition.split(" = ", 1) + key = key.strip() + value = value.strip() + state_value = state.get(key) + if value == "true": + return state_value is True + elif value == "false": + return state_value is False + else: + return str(state_value) == value + + if " < " in condition: + key, value = condition.split(" < ", 1) + key = key.strip() + value = value.strip().strip("'\"") + state_value = state.get(key, "") + return str(state_value) < value + + return False + + +def get_next_step(step, state): + """Determine the next step based on transitions and current state.""" + transitions = step.get("transitions", []) + for trans in transitions: + when = trans.get("when") + if when: + if evaluate_condition(when, state): + return trans.get("next") + else: + return trans.get("next") + return None + + +def walk_flow(flow_data, overrides=None): + """Walk through the flow, return the state object.""" + state = {} + overrides = overrides or {} + steps = flow_data.get("steps", {}) + current = flow_data.get("flow", {}).get("start") + visited_steps = [] + + while current: + step = steps.get(current) + if not step: + break + + visited_steps.append(current) + state_key = step.get("state_key") + + if state_key: + if state_key in overrides: + value = overrides[state_key] + else: + value = get_default_value(step) + state[state_key] = value + + current = get_next_step(step, state) + + return state, visited_steps + + +def evaluate_when_condition(when_expr, state): + """ + Evaluate a 'when' condition from the docs manifest against the state. + Supports: equality, 'in' arrays, 'and' conditions. + """ + if not when_expr: + return False + + # Use jsonata for complex expressions + try: + expr = jsonata.Jsonata(when_expr) + result = expr.evaluate(state) + return bool(result) + except Exception as e: + # Fallback to simple parsing for basic cases + return evaluate_simple_when(when_expr, state) + + +def evaluate_simple_when(when_expr, state): + """Simple fallback parser for when expressions.""" + # Handle "and" conditions + if " and " in when_expr: + parts = when_expr.split(" and ") + return all(evaluate_simple_when(p.strip(), state) for p in parts) + + # Handle "in" conditions: "platform in ['docker-compose', 'podman-compose']" + in_match = re.match(r"(\w+)\s+in\s+\[([^\]]+)\]", when_expr) + if in_match: + key = in_match.group(1) + values_str = in_match.group(2) + values = [v.strip().strip("'\"") for v in values_str.split(",")] + return state.get(key) in values + + # Handle equality: "platform = 'docker-compose'" + eq_match = re.match(r"(\w+)\s*=\s*['\"]([^'\"]+)['\"]", when_expr) + if eq_match: + key = eq_match.group(1) + value = eq_match.group(2) + return state.get(key) == value + + return False + + +def assemble_docs(state, manifest): + """ + Assemble documentation for a given state. + Returns (success, matched_instructions, errors). + """ + instructions = manifest.get("documentation", {}).get("instructions", []) + matched = [] + errors = [] + + for instr in instructions: + # Check if instruction applies + always = instr.get("always", False) + when = instr.get("when") + + applies = always or (when and evaluate_when_condition(when, state)) + + if applies: + file_path = instr.get("file") + content = load_doc_file(file_path) + + if content is None: + errors.append(f"Missing file: {file_path}") + matched.append({ + "id": instr.get("id"), + "goal": instr.get("goal"), + "file": file_path, + "found": False + }) + else: + matched.append({ + "id": instr.get("id"), + "goal": instr.get("goal"), + "file": file_path, + "found": True, + "content_length": len(content) + }) + + success = len(errors) == 0 and len(matched) > 0 + return success, matched, errors + + +def collect_fields_for_path(flow_data, overrides=None): + """Collect all fields and their possible values for a given path.""" + fields = [] + steps = flow_data.get("steps", {}) + _, visited = walk_flow(flow_data, overrides=overrides) + + for step_name in visited: + step = steps.get(step_name, {}) + state_key = step.get("state_key") + if state_key: + options = get_all_options(step) + default = get_default_value(step) + if len(options) > 1: + fields.append((step_name, state_key, options, default)) + + return fields + + +def run_single_test(flow_data, manifest, overrides, description, test_num, results): + """Run a single documentation assembly test.""" + print("-" * 70) + print(f"Test {test_num}: {description}") + print("-" * 70) + + state, _ = walk_flow(flow_data, overrides=overrides) + success, matched, errors = assemble_docs(state, manifest) + + result = { + "test": test_num, + "description": description, + "overrides": overrides, + "state": state, + "matched_count": len(matched), + "matched": matched, + "success": success + } + + if errors: + result["errors"] = errors + + results.append(result) + + print(f"State: {json.dumps({k: v for k, v in state.items() if not k.startswith('ocr') and not k.startswith('embed')}, indent=2)}") + print(f"Matched instructions: {len(matched)}") + for m in matched: + status = "OK" if m["found"] else "MISSING" + print(f" - [{status}] {m['goal']} ({m['file']})") + + if errors: + print(f"ERRORS: {errors}") + else: + print("Docs: OK") + print() + + return test_num + 1 + + +def run_test_matrix(flow_data, manifest): + """Run the documentation test matrix.""" + baseline_fields = collect_fields_for_path(flow_data) + baseline_keys = {f[1] for f in baseline_fields} + + print("=" * 70) + print("DOCUMENTATION TEST MATRIX") + print("=" * 70) + print() + print(f"Found {len(baseline_fields)} fields with multiple options on baseline path:") + for step_name, state_key, options, default in baseline_fields: + print(f" - {state_key}: {len(options)} options (default: {default})") + print() + + results = [] + test_num = 1 + + # Baseline test + test_num = run_single_test( + flow_data, manifest, + overrides={}, + description="BASELINE (all defaults)", + test_num=test_num, + results=results + ) + + # Test each field variation + for step_name, state_key, options, default in baseline_fields: + for option in options: + if option == default: + continue + + overrides = {state_key: option} + test_num = run_single_test( + flow_data, manifest, + overrides=overrides, + description=f"{state_key} = {option}", + test_num=test_num, + results=results + ) + + # Check for unlocked fields + unlocked_fields = collect_fields_for_path(flow_data, overrides=overrides) + unlocked_keys = {f[1] for f in unlocked_fields} + new_keys = unlocked_keys - baseline_keys + + if new_keys: + for uf_step, uf_key, uf_options, uf_default in unlocked_fields: + if uf_key not in new_keys: + continue + for uf_option in uf_options: + if uf_option == uf_default: + continue + + combined_overrides = {state_key: option, uf_key: uf_option} + test_num = run_single_test( + flow_data, manifest, + overrides=combined_overrides, + description=f"{state_key} = {option}, {uf_key} = {uf_option}", + test_num=test_num, + results=results + ) + + return results + + +def main(): + parser = argparse.ArgumentParser( + description="Test harness for documentation assembly" + ) + parser.add_argument( + "--matrix", "-m", + action="store_true", + help="Run test matrix (each field with all options)" + ) + args = parser.parse_args() + + flow_data = load_flow() + manifest = load_docs_manifest() + + if args.matrix: + results = run_test_matrix(flow_data, manifest) + + # Summary + print("=" * 70) + print("SUMMARY") + print("=" * 70) + print() + passed = [r for r in results if r["success"]] + failed = [r for r in results if not r["success"]] + print(f"Total tests: {len(results)}") + print(f"Passed: {len(passed)}") + print(f"Failed: {len(failed)}") + + if failed: + print() + print("Failed tests:") + for r in failed: + print(f" - Test {r['test']}: {r['description']}") + if "errors" in r: + for err in r["errors"]: + print(f" Error: {err}") + else: + # Single test with defaults + print("=" * 60) + print("Documentation Assembly Test - Default Options") + print("=" * 60) + print() + + state, _ = walk_flow(flow_data) + success, matched, errors = assemble_docs(state, manifest) + + print(f"State: {json.dumps(state, indent=2)}") + print() + print(f"Matched {len(matched)} instructions:") + for m in matched: + status = "OK" if m["found"] else "MISSING" + print(f" [{status}] {m['goal']}") + print(f" File: {m['file']}") + print() + + if errors: + print(f"Errors: {errors}") + else: + print("All documentation files found!") + + +if __name__ == "__main__": + main() diff --git a/ai-context/trustgraph-templates/tests/README.md b/ai-context/trustgraph-templates/tests/README.md new file mode 100644 index 00000000..064c8237 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/README.md @@ -0,0 +1,189 @@ +# TrustGraph Configurator Test Suite + +Comprehensive pytest-based test suite for trustgraph-configurator. + +## Installation + +Install with development dependencies: + +```bash +pip install -e .[dev] +``` + +## Running Tests + +```bash +# All tests +pytest + +# Specific category +pytest tests/unit/ +pytest tests/integration/ +pytest tests/validation/ + +# By marker +pytest -m unit +pytest -m integration +pytest -m validation + +# Specific test file +pytest tests/unit/test_generator.py + +# Parallel execution (faster) +pytest -n auto + +# Verbose output +pytest -v + +# Stop on first failure +pytest -x + +# With coverage +pytest --cov=trustgraph_configurator --cov-report=html +``` + +## Test Structure + +``` +tests/ +β”œβ”€β”€ conftest.py # Shared fixtures +β”œβ”€β”€ unit/ # Unit tests for Python modules +β”‚ β”œβ”€β”€ test_generator.py +β”‚ β”œβ”€β”€ test_packager.py +β”‚ β”œβ”€β”€ test_api.py +β”‚ └── test_run.py +β”œβ”€β”€ integration/ # Full workflow tests +β”‚ β”œβ”€β”€ test_compilation.py # Template compilation matrix +β”‚ β”œβ”€β”€ test_cli.py # CLI interface tests +β”‚ └── test_errors.py # Error handling tests +β”œβ”€β”€ validation/ # Output validation tests +β”‚ β”œβ”€β”€ test_syntax.py # Syntax validation +β”‚ β”œβ”€β”€ test_schema.py # Schema validation +β”‚ β”œβ”€β”€ test_semantics_k8s.py +β”‚ β”œβ”€β”€ test_semantics_docker.py +β”‚ └── test_semantics_tg.py +β”œβ”€β”€ validators/ # Validation helper modules +β”‚ β”œβ”€β”€ kubernetes.py +β”‚ β”œβ”€β”€ docker_compose.py +β”‚ └── trustgraph.py +β”œβ”€β”€ schemas/ # JSON schemas +β”‚ β”œβ”€β”€ trustgraph-config.schema.json +β”‚ β”œβ”€β”€ kubernetes-resource.schema.json +β”‚ └── docker-compose.schema.json +└── configs/ # Test input configs + β”œβ”€β”€ minimal.json + β”œβ”€β”€ complex-rag.json + β”œβ”€β”€ multi-service.json + └── cloud-aws.json +``` + +## Test Categories + +### Unit Tests (`tests/unit/`) +Test individual Python modules in isolation: +- Generator: Jsonnet template processing +- Packager: Configuration assembly and zip creation +- API: Template listing and version resolution +- Run: CLI entry point and argument parsing + +### Integration Tests (`tests/integration/`) +Test full workflow end-to-end: +- **Compilation**: Template compilation across all version/platform/config combinations (192 tests) +- **CLI**: Command line interface functionality +- **Errors**: Error handling and reporting + +### Validation Tests (`tests/validation/`) +Verify correctness of generated outputs: +- **Syntax**: JSON/YAML parsing validation +- **Schema**: JSON Schema compliance +- **Semantics**: Cross-references, consistency checks + +## Test Matrix + +Integration tests cover: +- **Versions**: 1.6, 1.7, 1.8 +- **Platforms**: docker-compose, podman-compose, minikube-k8s, gcp-k8s, aks-k8s, eks-k8s, scw-k8s, ovh-k8s +- **Configs**: minimal, complex-rag, multi-service, cloud-aws + +Total: 3 versions Γ— 8 platforms Γ— 4 configs Γ— 2 outputs = 192 test combinations + +## Validation Layers + +### Syntax Validation +- JSON parsing with `json.loads()` +- YAML parsing with `yaml.safe_load()` + +### Schema Validation +- TrustGraph config against `trustgraph-config.schema.json` +- Docker Compose against `docker-compose.schema.json` +- Kubernetes resources against `kubernetes-resource.schema.json` + +### Semantic Validation + +**Kubernetes:** +- Deployment selectors match pod labels +- Service selectors match deployment labels +- volumeMounts reference defined volumes +- ConfigMap/Secret references exist +- Service targetPorts match container ports + +**Docker Compose:** +- depends_on references valid services +- Volume names are defined +- Network references are valid +- No port conflicts + +**TrustGraph Config:** +- Service references are valid +- Parameter types are reasonable +- Storage backends are consistent +- LLM configuration is present + +## Fixtures + +Available in `conftest.py`: +- `test_config_dir`: Path to test configs +- `test_configs`: Loaded test configurations +- `temp_output_dir`: Temporary directory for outputs +- `run_configurator`: Function to execute configurator +- `mock_config_file`: Create temporary config files + +## CI/CD + +Tests run automatically on pull requests via GitHub Actions. + +See `.github/workflows/pull-request.yaml` for CI configuration. + +## Development + +### Adding New Tests + +1. Create test file in appropriate directory +2. Use appropriate markers (`@pytest.mark.unit`, etc.) +3. Use fixtures from `conftest.py` +4. Follow naming convention: `test_*.py`, `test_*()` functions + +### Adding New Validation + +1. Add validation logic to `tests/validators/` +2. Create corresponding tests in `tests/validation/` +3. Update schemas in `tests/schemas/` if needed + +## Troubleshooting + +**Test failures:** +- Check stderr output for error messages +- Run with `-v` for verbose output +- Run with `--tb=long` for full tracebacks + +**Import errors:** +- Ensure package is installed: `pip install -e .[dev]` +- Check PYTHONPATH includes project root + +**Slow tests:** +- Use `-n auto` for parallel execution +- Run specific test subsets instead of full suite + +## Documentation + +See `docs/tech-specs/tests.md` for detailed test specification. diff --git a/ai-context/trustgraph-templates/tests/configs/cloud-aws.json b/ai-context/trustgraph-templates/tests/configs/cloud-aws.json new file mode 100644 index 00000000..f09029e1 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/configs/cloud-aws.json @@ -0,0 +1,26 @@ +[ + { + "name": "bedrock", + "parameters": { + "model": "anthropic.claude-3-sonnet-20240229-v1:0" + } + }, + { + "name": "embeddings-hf", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tests/configs/complex-rag.json b/ai-context/trustgraph-templates/tests/configs/complex-rag.json new file mode 100644 index 00000000..ca93a4dd --- /dev/null +++ b/ai-context/trustgraph-templates/tests/configs/complex-rag.json @@ -0,0 +1,42 @@ +[ + { + "name": "openai", + "parameters": { + "model": "gpt-4", + "temperature": 0.7 + } + }, + { + "name": "embeddings-hf", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "triple-store-neo4j", + "parameters": {} + }, + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "trustgraph-base", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tests/configs/minimal.json b/ai-context/trustgraph-templates/tests/configs/minimal.json new file mode 100644 index 00000000..61e76a3f --- /dev/null +++ b/ai-context/trustgraph-templates/tests/configs/minimal.json @@ -0,0 +1,23 @@ +[ + { + "name": "openai", + "parameters": { + "model": "gpt-3.5-turbo", + "temperature": 0.7 + } + }, + { + "name": "embeddings-hf", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "trustgraph-base", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tests/configs/multi-service.json b/ai-context/trustgraph-templates/tests/configs/multi-service.json new file mode 100644 index 00000000..f475fb75 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/configs/multi-service.json @@ -0,0 +1,38 @@ +[ + { + "name": "ollama", + "parameters": { + "model": "llama2" + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "BAAI/bge-small-en-v1.5" + } + }, + { + "name": "vector-store-milvus", + "parameters": {} + }, + { + "name": "triple-store-memgraph", + "parameters": {} + }, + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tests/conftest.py b/ai-context/trustgraph-templates/tests/conftest.py new file mode 100644 index 00000000..76a04905 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/conftest.py @@ -0,0 +1,125 @@ +""" +Pytest configuration and shared fixtures for trustgraph-configurator tests. +""" + +import pytest + +# ============================================================================= +# Version Configuration - Update these when adding new template versions +# ============================================================================= +TESTED_VERSIONS = ["1.8", "1.9", "2.0"] +PRIMARY_VERSION = "1.9" # Used when only one version is tested +import sys +import json +import tempfile +import shutil +from pathlib import Path + + +@pytest.fixture(scope="session") +def test_config_dir(): + """Path to the test configurations directory.""" + return Path(__file__).parent / "configs" + + +@pytest.fixture(scope="session") +def test_configs(test_config_dir): + """Dictionary of loaded test configurations.""" + configs = {} + for config_file in test_config_dir.glob("*.json"): + with open(config_file) as f: + configs[config_file.name] = json.load(f) + return configs + + +@pytest.fixture +def temp_output_dir(): + """Temporary directory for test outputs.""" + temp_dir = tempfile.mkdtemp() + yield Path(temp_dir) + shutil.rmtree(temp_dir) + + +@pytest.fixture +def run_configurator(monkeypatch, capsys): + """ + Fixture to run configurator with given arguments. + + Usage: + stdout, stderr, exit_code = run_configurator(['-t', '1.8', '-p', 'docker-compose', ...]) + + Returns: + tuple: (stdout, stderr, exit_code) + """ + def _run(args): + from trustgraph_configurator import run + + # Set sys.argv with the command and arguments + monkeypatch.setattr(sys, 'argv', ['tg-build-deployment'] + args) + + exit_code = 0 + try: + run() # run is already the function, not a module + except SystemExit as e: + exit_code = e.code if e.code is not None else 0 + + # Capture output + captured = capsys.readouterr() + return captured.out, captured.err, exit_code + + return _run + + +@pytest.fixture(scope="session") +def golden_dir(): + """Path to the golden files directory.""" + return Path(__file__).parent / "golden" + + +@pytest.fixture(scope="session") +def test_versions(): + """List of template versions to test.""" + return TESTED_VERSIONS + + +@pytest.fixture(scope="session") +def primary_version(): + """Primary version for tests that only need one version.""" + return PRIMARY_VERSION + + +@pytest.fixture(scope="session") +def test_platforms(): + """List of platforms to test.""" + return [ + "docker-compose", + "podman-compose", + "minikube-k8s", + "gcp-k8s", + "aks-k8s", + "eks-k8s", + "scw-k8s", + "ovh-k8s", + ] + + +@pytest.fixture(scope="session") +def test_config_names(): + """List of test configuration file names.""" + return [ + "minimal.json", + "complex-rag.json", + "multi-service.json", + "cloud-aws.json", + ] + + +@pytest.fixture +def mock_config_file(tmp_path): + """Create a temporary config file for testing.""" + def _create(config_data): + config_file = tmp_path / "test_config.json" + with open(config_file, 'w') as f: + json.dump(config_data, f) + return str(config_file) + return _create diff --git a/ai-context/trustgraph-templates/tests/integration/__init__.py b/ai-context/trustgraph-templates/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai-context/trustgraph-templates/tests/integration/test_cli.py b/ai-context/trustgraph-templates/tests/integration/test_cli.py new file mode 100644 index 00000000..1666254c --- /dev/null +++ b/ai-context/trustgraph-templates/tests/integration/test_cli.py @@ -0,0 +1,89 @@ +""" +Integration tests for CLI interface. +""" + +import pytest +import subprocess + +from conftest import TESTED_VERSIONS + + +@pytest.mark.integration +class TestCLIInterface: + """Tests for CLI command line interface.""" + + def test_cli_executable_help(self): + """Test that CLI executable --help works.""" + result = subprocess.run( + ['tg-build-deployment', '--help'], + capture_output=True, + text=True + ) + assert result.returncode == 0 + assert 'usage' in result.stdout.lower() + + def test_cli_executable_exists(self): + """Test that tg-build-deployment is in PATH.""" + result = subprocess.run( + ['which', 'tg-build-deployment'], + capture_output=True, + text=True + ) + assert result.returncode == 0 + + def test_output_modes(self, run_configurator, test_config_dir, primary_version): + """Test -O and -R output modes.""" + config_file = str(test_config_dir / "minimal.json") + + # Test -O mode + stdout_o, _, code_o = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-O' + ]) + assert code_o == 0 + assert len(stdout_o) > 0 + + # Test -R mode + stdout_r, _, code_r = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-R' + ]) + assert code_r == 0 + assert len(stdout_r) > 0 + + # Outputs should be different + assert stdout_o != stdout_r + + def test_platform_argument(self, run_configurator, test_config_dir, primary_version): + """Test -p/--platform argument.""" + config_file = str(test_config_dir / "minimal.json") + + for platform in ['docker-compose', 'minikube-k8s']: + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', platform, + '-i', config_file, + '--latest-stable', + '-O' + ]) + assert code == 0, f"Failed for platform {platform}" + + def test_template_argument(self, run_configurator, test_config_dir): + """Test -t/--template argument.""" + config_file = str(test_config_dir / "minimal.json") + + for template in TESTED_VERSIONS: + stdout, stderr, code = run_configurator([ + '-t', template, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-O' + ]) + assert code == 0, f"Failed for template {template}" diff --git a/ai-context/trustgraph-templates/tests/integration/test_compilation.py b/ai-context/trustgraph-templates/tests/integration/test_compilation.py new file mode 100644 index 00000000..e38f27d2 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/integration/test_compilation.py @@ -0,0 +1,163 @@ +""" +Integration tests for template compilation across all combinations. +""" + +import pytest +import json +import yaml + +from conftest import TESTED_VERSIONS + + +@pytest.mark.integration +@pytest.mark.parametrize("version", TESTED_VERSIONS) +@pytest.mark.parametrize("platform", [ + "docker-compose", + "podman-compose", + "minikube-k8s", + "gcp-k8s", + "aks-k8s", + "eks-k8s", + "scw-k8s", + "ovh-k8s", +]) +@pytest.mark.parametrize("config", [ + "minimal.json", + "complex-rag.json", + "multi-service.json", + "cloud-aws.json", +]) +def test_tg_config_generation(version, platform, config, run_configurator, test_config_dir): + """Test TrustGraph config generation for all combinations.""" + config_file = str(test_config_dir / config) + + stdout, stderr, code = run_configurator([ + '-t', version, + '-p', platform, + '-i', config_file, + '--latest-stable', + '-O' + ]) + + # Should succeed + assert code == 0, f"Failed for {version}/{platform}/{config}: {stderr}" + + # Should output valid JSON + try: + tg_config = json.loads(stdout) + except json.JSONDecodeError as e: + pytest.fail(f"Invalid JSON output for {version}/{platform}/{config}: {e}") + + # Basic structure checks + assert isinstance(tg_config, (dict, list)), "TrustGraph config should be dict or list" + + +@pytest.mark.integration +@pytest.mark.parametrize("version", TESTED_VERSIONS) +@pytest.mark.parametrize("platform", [ + "docker-compose", + "podman-compose", + "minikube-k8s", + "gcp-k8s", + "aks-k8s", + "eks-k8s", + "scw-k8s", + "ovh-k8s", +]) +@pytest.mark.parametrize("config", [ + "minimal.json", + "complex-rag.json", + "multi-service.json", + "cloud-aws.json", +]) +def test_resources_generation(version, platform, config, run_configurator, test_config_dir): + """Test platform resources generation for all combinations.""" + config_file = str(test_config_dir / config) + + stdout, stderr, code = run_configurator([ + '-t', version, + '-p', platform, + '-i', config_file, + '--latest-stable', + '-R' + ]) + + # Should succeed + assert code == 0, f"Failed for {version}/{platform}/{config}: {stderr}" + + # Should output valid YAML + try: + resources = yaml.safe_load(stdout) + except yaml.YAMLError as e: + pytest.fail(f"Invalid YAML output for {version}/{platform}/{config}: {e}") + + # Basic structure checks + if platform in ["docker-compose", "podman-compose"]: + assert "services" in resources, "Docker Compose should have services" + else: + # Kubernetes resources + assert resources is not None, "K8s resources should not be empty" + + +@pytest.mark.integration +def test_compilation_minimal_docker_compose(run_configurator, test_config_dir, primary_version): + """Smoke test: minimal config on docker-compose.""" + config_file = str(test_config_dir / "minimal.json") + + # Test TG config + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-O' + ]) + + assert code == 0 + tg_config = json.loads(stdout) + assert tg_config is not None + + # Test resources + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + resources = yaml.safe_load(stdout) + assert "services" in resources + + +@pytest.mark.integration +def test_compilation_minimal_k8s(run_configurator, test_config_dir, primary_version): + """Smoke test: minimal config on k8s.""" + config_file = str(test_config_dir / "minimal.json") + + # Test TG config + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'minikube-k8s', + '-i', config_file, + '--latest-stable', + '-O' + ]) + + assert code == 0 + tg_config = json.loads(stdout) + assert tg_config is not None + + # Test resources + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'minikube-k8s', + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + resources = yaml.safe_load(stdout) + assert resources is not None diff --git a/ai-context/trustgraph-templates/tests/integration/test_errors.py b/ai-context/trustgraph-templates/tests/integration/test_errors.py new file mode 100644 index 00000000..ad13ee4f --- /dev/null +++ b/ai-context/trustgraph-templates/tests/integration/test_errors.py @@ -0,0 +1,97 @@ +""" +Integration tests for error handling. +""" + +import pytest +import json + + +@pytest.mark.integration +class TestErrorHandling: + """Tests for error handling and reporting.""" + + def test_nonexistent_config_file(self, run_configurator, primary_version): + """Test error when config file doesn't exist.""" + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', '/nonexistent/config.json', + '--latest-stable', + '-O' + ]) + assert code == 1 + assert len(stderr) > 0 # Error should be in stderr + + def test_invalid_json_config(self, run_configurator, tmp_path, primary_version): + """Test error when config file has invalid JSON.""" + invalid_config = tmp_path / "invalid.json" + invalid_config.write_text("{ invalid json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', str(invalid_config), + '--latest-stable', + '-O' + ]) + assert code == 1 + + def test_invalid_platform(self, run_configurator, test_config_dir, primary_version): + """Test error when platform is invalid.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'nonexistent-platform', + '-i', config_file, + '--latest-stable', + '-R' # Use -R to trigger platform-specific generation + ]) + assert code == 1 + + def test_invalid_template_version(self, run_configurator, test_config_dir): + """Test error when template version doesn't exist.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', '999.999', + '-p', 'docker-compose', + '-i', config_file, + '-O' + ]) + assert code == 1 + + def test_malformed_config_structure(self, run_configurator, tmp_path, primary_version): + """Test error when config structure is invalid.""" + # Valid JSON but wrong structure + invalid_config = tmp_path / "bad_structure.json" + invalid_config.write_text('{"wrong": "structure"}') + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', str(invalid_config), + '--latest-stable', + '-O' + ]) + # May succeed with warning or fail - either is acceptable + # The important thing is it doesn't crash + assert code in [0, 1] + + def test_missing_required_args(self, run_configurator): + """Test error when required arguments are missing.""" + # Missing template + stdout, stderr, code = run_configurator([ + '-p', 'docker-compose', + '-O' + ]) + assert code == 1 + + def test_error_goes_to_stderr(self, run_configurator): + """Test that errors are written to stderr, not stdout.""" + stdout, stderr, code = run_configurator([ + '-i', '/nonexistent/config.json' + ]) + assert code == 1 + # Errors should be in stderr + assert len(stderr) > 0 or 'Exception' in stderr or code == 1 diff --git a/ai-context/trustgraph-templates/tests/schemas/docker-compose.schema.json b/ai-context/trustgraph-templates/tests/schemas/docker-compose.schema.json new file mode 100644 index 00000000..ea7e6a27 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/schemas/docker-compose.schema.json @@ -0,0 +1,103 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Docker Compose Configuration", + "description": "Basic schema for Docker Compose files", + "type": "object", + "required": ["services"], + "properties": { + "version": { + "type": "string", + "description": "Docker Compose version" + }, + "services": { + "type": "object", + "description": "Service definitions", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "properties": { + "image": { + "type": "string", + "description": "Docker image" + }, + "build": { + "oneOf": [ + {"type": "string"}, + {"type": "object"} + ], + "description": "Build configuration" + }, + "ports": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + {"type": "integer"} + ] + } + }, + "volumes": { + "type": "array", + "items": { + "type": "string" + } + }, + "environment": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "oneOf": [ + {"type": "string"}, + {"type": "number"}, + {"type": "boolean"} + ] + } + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] + }, + "depends_on": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "object" + } + ] + }, + "networks": { + "oneOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "object" + } + ] + } + } + } + }, + "volumes": { + "type": "object", + "description": "Named volumes" + }, + "networks": { + "type": "object", + "description": "Network definitions" + } + } +} diff --git a/ai-context/trustgraph-templates/tests/schemas/kubernetes-resource.schema.json b/ai-context/trustgraph-templates/tests/schemas/kubernetes-resource.schema.json new file mode 100644 index 00000000..943096ab --- /dev/null +++ b/ai-context/trustgraph-templates/tests/schemas/kubernetes-resource.schema.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Kubernetes Resource", + "description": "Basic schema for Kubernetes resources", + "type": "object", + "required": ["apiVersion", "kind", "metadata"], + "properties": { + "apiVersion": { + "type": "string", + "description": "Kubernetes API version" + }, + "kind": { + "type": "string", + "description": "Resource kind", + "enum": ["Deployment", "Service", "ConfigMap", "Secret", "PersistentVolumeClaim", "PersistentVolume", "Namespace", "StorageClass"] + }, + "metadata": { + "type": "object", + "required": ["name"], + "properties": { + "name": { + "type": "string", + "description": "Resource name" + }, + "namespace": { + "type": "string", + "description": "Resource namespace" + }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "spec": { + "type": "object", + "description": "Resource specification" + } + } +} diff --git a/ai-context/trustgraph-templates/tests/schemas/trustgraph-config.schema.json b/ai-context/trustgraph-templates/tests/schemas/trustgraph-config.schema.json new file mode 100644 index 00000000..61082e44 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/schemas/trustgraph-config.schema.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TrustGraph Configuration Output", + "description": "Schema for generated TrustGraph configuration", + "type": "object", + "properties": { + "collection": { + "type": "object", + "description": "Collection definitions" + }, + "tools": { + "type": "object", + "description": "Tool definitions" + } + } +} diff --git a/ai-context/trustgraph-templates/tests/unit/__init__.py b/ai-context/trustgraph-templates/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai-context/trustgraph-templates/tests/unit/test_api.py b/ai-context/trustgraph-templates/tests/unit/test_api.py new file mode 100644 index 00000000..69a18cb9 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/unit/test_api.py @@ -0,0 +1,38 @@ +""" +Unit tests for Index class. +""" + +import pytest +from trustgraph_configurator import Index + + +@pytest.mark.unit +class TestAPI: + """Tests for the Index class.""" + + def test_get_templates_returns_list(self): + """Test that get_templates returns a list.""" + templates = Index.get_templates() + assert isinstance(templates, list) + assert len(templates) > 0 + + def test_templates_have_required_fields(self): + """Test that templates have name and version fields.""" + templates = Index.get_templates() + for template in templates: + assert hasattr(template, 'name') + assert hasattr(template, 'version') + + def test_get_latest_returns_template(self): + """Test that get_latest returns a template.""" + latest = Index.get_latest() + assert latest is not None + assert hasattr(latest, 'name') + assert hasattr(latest, 'version') + + def test_get_latest_stable_returns_template(self): + """Test that get_latest_stable returns a template.""" + latest_stable = Index.get_latest_stable() + assert latest_stable is not None + assert hasattr(latest_stable, 'name') + assert hasattr(latest_stable, 'version') diff --git a/ai-context/trustgraph-templates/tests/unit/test_generator.py b/ai-context/trustgraph-templates/tests/unit/test_generator.py new file mode 100644 index 00000000..3431153b --- /dev/null +++ b/ai-context/trustgraph-templates/tests/unit/test_generator.py @@ -0,0 +1,101 @@ +""" +Unit tests for Generator class. +""" + +import pytest +import json +from trustgraph_configurator.generator import Generator + + +@pytest.mark.unit +class TestGenerator: + """Tests for the Generator class.""" + + def test_simple_jsonnet(self): + """Test processing simple jsonnet.""" + def mock_fetch(base, rel): + return "", "" + + generator = Generator(mock_fetch) + result = generator.process('{ foo: "bar" }') + + assert isinstance(result, dict) + assert result["foo"] == "bar" + + def test_jsonnet_with_variables(self): + """Test processing jsonnet with variables.""" + def mock_fetch(base, rel): + return "", "" + + generator = Generator(mock_fetch) + jsonnet_code = ''' + local name = "test"; + { + name: name, + value: 42 + } + ''' + result = generator.process(jsonnet_code) + + assert result["name"] == "test" + assert result["value"] == 42 + + def test_jsonnet_with_array(self): + """Test processing jsonnet that returns array.""" + def mock_fetch(base, rel): + return "", "" + + generator = Generator(mock_fetch) + jsonnet_code = '[1, 2, 3, { foo: "bar" }]' + result = generator.process(jsonnet_code) + + assert isinstance(result, list) + assert len(result) == 4 + assert result[0] == 1 + assert result[3]["foo"] == "bar" + + def test_invalid_jsonnet(self): + """Test that invalid jsonnet raises exception.""" + def mock_fetch(base, rel): + return "", "" + + generator = Generator(mock_fetch) + + with pytest.raises(Exception): + generator.process('{ invalid jsonnet') + + def test_jsonnet_with_functions(self): + """Test processing jsonnet with functions.""" + def mock_fetch(base, rel): + return "", "" + + generator = Generator(mock_fetch) + jsonnet_code = ''' + local double(x) = x * 2; + { + value: double(21) + } + ''' + result = generator.process(jsonnet_code) + + assert result["value"] == 42 + + def test_fetch_callback_is_used(self): + """Test that fetch callback is called for imports.""" + fetch_called = [] + + def mock_fetch(base, rel): + fetch_called.append((base, rel)) + # Return simple jsonnet that defines a variable (as bytes) + return "config", b'{ imported: true }' + + generator = Generator(mock_fetch) + jsonnet_code = ''' + local config = import "config.jsonnet"; + config + ''' + + result = generator.process(jsonnet_code) + + assert len(fetch_called) > 0 + assert result["imported"] is True diff --git a/ai-context/trustgraph-templates/tests/unit/test_packager.py b/ai-context/trustgraph-templates/tests/unit/test_packager.py new file mode 100644 index 00000000..06f53cd0 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/unit/test_packager.py @@ -0,0 +1,60 @@ +""" +Unit tests for Packager class. +""" + +import pytest +from trustgraph_configurator.packager import Packager + + +@pytest.mark.unit +class TestPackager: + """Tests for the Packager class.""" + + def test_init_with_latest_stable(self): + """Test initialization with latest_stable flag.""" + packager = Packager( + version=None, + template=None, + platform="docker-compose", + latest=False, + latest_stable=True + ) + assert packager.version is not None + assert packager.template is not None + + def test_init_with_template(self): + """Test initialization with specific template.""" + packager = Packager( + version="1.8.12", + template="1.8", + platform="docker-compose", + latest=False, + latest_stable=False + ) + assert packager.version == "1.8.12" + assert packager.template == "1.8" + assert packager.platform == "docker-compose" + + def test_invalid_platform_raises_error(self): + """Test that invalid platform raises error during generation.""" + packager = Packager( + version="1.8.12", + template="1.8", + platform="invalid-platform", + latest=False, + latest_stable=False + ) + + with pytest.raises(RuntimeError, match="Bad platform"): + packager.generate('[{"name": "test", "parameters": {}}]') + + def test_init_without_template_raises_error(self): + """Test that initialization without template/latest raises error.""" + with pytest.raises(RuntimeError, match="You must"): + Packager( + version=None, + template=None, + platform="docker-compose", + latest=False, + latest_stable=False + ) diff --git a/ai-context/trustgraph-templates/tests/unit/test_run.py b/ai-context/trustgraph-templates/tests/unit/test_run.py new file mode 100644 index 00000000..25f68c23 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/unit/test_run.py @@ -0,0 +1,62 @@ +""" +Unit tests for run module (CLI entry point). +""" + +import pytest +import sys + + +@pytest.mark.unit +class TestRun: + """Tests for the run module.""" + + def test_run_without_args_fails(self, run_configurator): + """Test that running without required args fails.""" + stdout, stderr, code = run_configurator([]) + assert code != 0 + + def test_run_with_help_succeeds(self, run_configurator): + """Test that help flag works.""" + stdout, stderr, code = run_configurator(['-h']) + assert code == 0 + + def test_run_with_invalid_platform_fails(self, run_configurator, test_config_dir, primary_version): + """Test that invalid platform fails during resource generation.""" + config_file = str(test_config_dir / "minimal.json") + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'invalid-platform', + '-i', config_file, + '--latest-stable', + '-R' # Use -R to trigger platform-specific generation + ]) + assert code == 1 + + def test_run_with_nonexistent_config_fails(self, run_configurator, primary_version): + """Test that nonexistent config file fails.""" + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', '/nonexistent/config.json', + '--latest-stable', + '-O' + ]) + assert code == 1 + + def test_exit_code_propagates(self, monkeypatch): + """Test that exit codes are properly set.""" + from trustgraph_configurator import run + + # Test successful exit (no exception) + # This would require a valid config, so we'll just test the error path + + # Test error exit + monkeypatch.setattr(sys, 'argv', [ + 'tg-build-deployment', + '-i', '/nonexistent/config.json' + ]) + + with pytest.raises(SystemExit) as exc_info: + run() # run is already the function + + assert exc_info.value.code == 1 diff --git a/ai-context/trustgraph-templates/tests/validation/__init__.py b/ai-context/trustgraph-templates/tests/validation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai-context/trustgraph-templates/tests/validation/test_schema.py b/ai-context/trustgraph-templates/tests/validation/test_schema.py new file mode 100644 index 00000000..10f585f3 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/validation/test_schema.py @@ -0,0 +1,111 @@ +""" +Schema validation tests for generated outputs. +""" + +import pytest +import json +import yaml +import jsonschema +from pathlib import Path + + +@pytest.fixture(scope="module") +def schemas_dir(): + """Path to schemas directory.""" + return Path(__file__).parent.parent / "schemas" + + +@pytest.mark.validation +def test_tg_config_matches_schema(run_configurator, test_config_dir, schemas_dir, primary_version): + """Test that TrustGraph config matches schema.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-O' + ]) + + assert code == 0 + + tg_config = json.loads(stdout) + schema_file = schemas_dir / "trustgraph-config.schema.json" + + with open(schema_file) as f: + schema = json.load(f) + + try: + jsonschema.validate(instance=tg_config, schema=schema) + except jsonschema.ValidationError as e: + pytest.fail(f"Schema validation failed: {e}") + + +@pytest.mark.validation +def test_docker_compose_matches_schema(run_configurator, test_config_dir, schemas_dir, primary_version): + """Test that Docker Compose output matches schema.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + + compose_data = yaml.safe_load(stdout) + schema_file = schemas_dir / "docker-compose.schema.json" + + with open(schema_file) as f: + schema = json.load(f) + + try: + jsonschema.validate(instance=compose_data, schema=schema) + except jsonschema.ValidationError as e: + pytest.fail(f"Schema validation failed: {e}") + + +@pytest.mark.validation +def test_kubernetes_resources_match_schema(run_configurator, test_config_dir, schemas_dir, primary_version): + """Test that Kubernetes resources match schema.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'minikube-k8s', + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + + resources = yaml.safe_load(stdout) + schema_file = schemas_dir / "kubernetes-resource.schema.json" + + with open(schema_file) as f: + schema = json.load(f) + + # Validate the resource (which might be a single resource, list, or K8s List) + if isinstance(resources, dict): + # Check if it's a Kubernetes List resource + if resources.get('kind') == 'List' and 'items' in resources: + resources_to_validate = resources['items'] + else: + resources_to_validate = [resources] + elif isinstance(resources, list): + resources_to_validate = resources + else: + pytest.fail(f"Unexpected resources type: {type(resources)}") + + for resource in resources_to_validate: + if not isinstance(resource, dict): + continue # Skip non-dict items + try: + jsonschema.validate(instance=resource, schema=schema) + except jsonschema.ValidationError as e: + pytest.fail(f"Schema validation failed for resource: {e}") diff --git a/ai-context/trustgraph-templates/tests/validation/test_semantics_docker.py b/ai-context/trustgraph-templates/tests/validation/test_semantics_docker.py new file mode 100644 index 00000000..70db38c0 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/validation/test_semantics_docker.py @@ -0,0 +1,78 @@ +""" +Semantic validation tests for Docker Compose resources. +""" + +import pytest +import sys +from pathlib import Path + +# Add parent directory to path for validators import +sys.path.insert(0, str(Path(__file__).parent.parent)) +from validators import docker_compose + + +@pytest.mark.validation +@pytest.mark.parametrize("config", ["minimal.json", "complex-rag.json"]) +def test_docker_compose_semantic_validation(config, run_configurator, test_config_dir, primary_version): + """Test semantic validation of Docker Compose resources.""" + config_file = str(test_config_dir / config) + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + + is_valid, errors = docker_compose.validate_docker_compose_manifest(stdout) + + if not is_valid: + error_msg = "\n".join(errors) + pytest.fail(f"Semantic validation failed for {config}:\n{error_msg}") + + +@pytest.mark.validation +def test_docker_compose_service_dependencies(run_configurator, test_config_dir, primary_version): + """Test that service dependencies reference valid services.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + + compose_data = docker_compose.parse_docker_compose_yaml(stdout) + errors = docker_compose.validate_service_dependencies(compose_data) + + if errors: + pytest.fail(f"Invalid service dependencies:\n" + "\n".join(errors)) + + +@pytest.mark.validation +def test_docker_compose_no_port_conflicts(run_configurator, test_config_dir, primary_version): + """Test that there are no port conflicts.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + + compose_data = docker_compose.parse_docker_compose_yaml(stdout) + errors = docker_compose.validate_port_conflicts(compose_data) + + if errors: + pytest.fail(f"Port conflicts detected:\n" + "\n".join(errors)) diff --git a/ai-context/trustgraph-templates/tests/validation/test_semantics_k8s.py b/ai-context/trustgraph-templates/tests/validation/test_semantics_k8s.py new file mode 100644 index 00000000..16eeb97e --- /dev/null +++ b/ai-context/trustgraph-templates/tests/validation/test_semantics_k8s.py @@ -0,0 +1,78 @@ +""" +Semantic validation tests for Kubernetes resources. +""" + +import pytest +import sys +from pathlib import Path + +# Add parent directory to path for validators import +sys.path.insert(0, str(Path(__file__).parent.parent)) +from validators import kubernetes + + +@pytest.mark.validation +@pytest.mark.parametrize("config", ["minimal.json", "complex-rag.json"]) +def test_k8s_semantic_validation(config, run_configurator, test_config_dir, primary_version): + """Test semantic validation of Kubernetes resources.""" + config_file = str(test_config_dir / config) + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'minikube-k8s', + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + + is_valid, errors = kubernetes.validate_kubernetes_manifest(stdout) + + if not is_valid: + error_msg = "\n".join(errors) + pytest.fail(f"Semantic validation failed for {config}:\n{error_msg}") + + +@pytest.mark.validation +def test_k8s_selector_labels_match(run_configurator, test_config_dir, primary_version): + """Test that Deployment selectors match pod labels.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'minikube-k8s', + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + + resources = kubernetes.parse_kubernetes_yaml(stdout) + errors = kubernetes.validate_selector_labels_match(resources) + + if errors: + pytest.fail(f"Selector/label mismatch:\n" + "\n".join(errors)) + + +@pytest.mark.validation +def test_k8s_volume_references(run_configurator, test_config_dir, primary_version): + """Test that volumeMounts reference defined volumes.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'minikube-k8s', + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + + resources = kubernetes.parse_kubernetes_yaml(stdout) + errors = kubernetes.validate_volume_references(resources) + + if errors: + pytest.fail(f"Invalid volume references:\n" + "\n".join(errors)) diff --git a/ai-context/trustgraph-templates/tests/validation/test_semantics_tg.py b/ai-context/trustgraph-templates/tests/validation/test_semantics_tg.py new file mode 100644 index 00000000..e97be793 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/validation/test_semantics_tg.py @@ -0,0 +1,83 @@ +""" +Semantic validation tests for TrustGraph configuration. +""" + +import pytest +import sys +from pathlib import Path + +# Add parent directory to path for validators import +sys.path.insert(0, str(Path(__file__).parent.parent)) +from validators import trustgraph + + +@pytest.mark.validation +@pytest.mark.parametrize("config", ["minimal.json", "complex-rag.json", "multi-service.json"]) +def test_tg_config_semantic_validation(config, run_configurator, test_config_dir, primary_version): + """Test semantic validation of TrustGraph configuration.""" + config_file = str(test_config_dir / config) + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-O' + ]) + + assert code == 0 + + is_valid, errors = trustgraph.validate_trustgraph_config(stdout) + + if not is_valid: + error_msg = "\n".join(errors) + # Some errors might be warnings, so we log them but don't necessarily fail + # Adjust this based on strictness requirements + if any("missing" in err.lower() or "required" in err.lower() for err in errors): + pytest.fail(f"Semantic validation failed for {config}:\n{error_msg}") + + +@pytest.mark.validation +def test_tg_config_has_llm(run_configurator, test_config_dir, primary_version): + """Test that TrustGraph config includes LLM provider.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-O' + ]) + + assert code == 0 + + tg_config = trustgraph.parse_trustgraph_config(stdout) + errors = trustgraph.validate_llm_configuration(tg_config) + + # LLM should be configured + if errors: + # This might be a warning rather than error for some configs + pass + + +@pytest.mark.validation +def test_tg_config_structure(run_configurator, test_config_dir, primary_version): + """Test that TrustGraph config has required structure.""" + config_file = str(test_config_dir / "minimal.json") + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', 'docker-compose', + '-i', config_file, + '--latest-stable', + '-O' + ]) + + assert code == 0 + + tg_config = trustgraph.parse_trustgraph_config(stdout) + errors = trustgraph.validate_required_structure(tg_config) + + if errors: + pytest.fail(f"Invalid TrustGraph config structure:\n" + "\n".join(errors)) diff --git a/ai-context/trustgraph-templates/tests/validation/test_syntax.py b/ai-context/trustgraph-templates/tests/validation/test_syntax.py new file mode 100644 index 00000000..c0ccb15e --- /dev/null +++ b/ai-context/trustgraph-templates/tests/validation/test_syntax.py @@ -0,0 +1,57 @@ +""" +Syntax validation tests for generated outputs. +""" + +import pytest +import json +import yaml + + +@pytest.mark.validation +@pytest.mark.parametrize("platform", ["docker-compose", "minikube-k8s"]) +@pytest.mark.parametrize("config", ["minimal.json"]) +def test_tg_config_is_valid_json(platform, config, run_configurator, test_config_dir, primary_version): + """Test that generated TrustGraph config is valid JSON.""" + config_file = str(test_config_dir / config) + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', platform, + '-i', config_file, + '--latest-stable', + '-O' + ]) + + assert code == 0 + + # Should parse as valid JSON + try: + parsed = json.loads(stdout) + assert parsed is not None + except json.JSONDecodeError as e: + pytest.fail(f"Invalid JSON: {e}") + + +@pytest.mark.validation +@pytest.mark.parametrize("platform", ["docker-compose", "minikube-k8s"]) +@pytest.mark.parametrize("config", ["minimal.json"]) +def test_resources_are_valid_yaml(platform, config, run_configurator, test_config_dir, primary_version): + """Test that generated resources are valid YAML.""" + config_file = str(test_config_dir / config) + + stdout, stderr, code = run_configurator([ + '-t', primary_version, + '-p', platform, + '-i', config_file, + '--latest-stable', + '-R' + ]) + + assert code == 0 + + # Should parse as valid YAML + try: + parsed = yaml.safe_load(stdout) + assert parsed is not None + except yaml.YAMLError as e: + pytest.fail(f"Invalid YAML: {e}") diff --git a/ai-context/trustgraph-templates/tests/validators/__init__.py b/ai-context/trustgraph-templates/tests/validators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ai-context/trustgraph-templates/tests/validators/docker_compose.py b/ai-context/trustgraph-templates/tests/validators/docker_compose.py new file mode 100644 index 00000000..f6405ff9 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/validators/docker_compose.py @@ -0,0 +1,242 @@ +""" +Docker Compose manifest semantic validation. +""" + +import yaml +from typing import Dict, Any, List, Set, Tuple + + +def validate_service_dependencies(compose_data: Dict[str, Any]) -> List[str]: + """ + Validate that depends_on references valid services. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + services = compose_data.get('services', {}) + service_names = set(services.keys()) + + for service_name, service_spec in services.items(): + depends_on = service_spec.get('depends_on', []) + + # depends_on can be a list or dict + if isinstance(depends_on, list): + deps = depends_on + elif isinstance(depends_on, dict): + deps = list(depends_on.keys()) + else: + continue + + for dep in deps: + if dep not in service_names: + errors.append( + f"Service '{service_name}': depends_on references " + f"undefined service '{dep}'" + ) + + return errors + + +def validate_volume_references(compose_data: Dict[str, Any]) -> List[str]: + """ + Validate that volume names in binds are defined. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + services = compose_data.get('services', {}) + defined_volumes = set(compose_data.get('volumes', {}).keys()) + + for service_name, service_spec in services.items(): + volumes = service_spec.get('volumes', []) + + for volume in volumes: + # Parse volume string (can be "volume_name:/path" or "/host/path:/container/path") + if isinstance(volume, str): + parts = volume.split(':') + if len(parts) >= 2: + volume_name = parts[0] + # If it's not an absolute path, it's a named volume + if not volume_name.startswith('/') and not volume_name.startswith('.'): + if volume_name not in defined_volumes: + errors.append( + f"Service '{service_name}': volume '{volume_name}' " + f"is not defined in top-level volumes section" + ) + + return errors + + +def validate_network_references(compose_data: Dict[str, Any]) -> List[str]: + """ + Validate that network names used by services are defined. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + services = compose_data.get('services', {}) + defined_networks = set(compose_data.get('networks', {}).keys()) + + # Add default network + defined_networks.add('default') + + for service_name, service_spec in services.items(): + networks = service_spec.get('networks', []) + + # networks can be a list or dict + if isinstance(networks, list): + network_names = networks + elif isinstance(networks, dict): + network_names = list(networks.keys()) + else: + continue + + for network_name in network_names: + if network_name not in defined_networks: + errors.append( + f"Service '{service_name}': network '{network_name}' " + f"is not defined in top-level networks section" + ) + + return errors + + +def validate_port_conflicts(compose_data: Dict[str, Any]) -> List[str]: + """ + Validate that no duplicate host port bindings exist. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + services = compose_data.get('services', {}) + used_ports: Dict[int, str] = {} + + for service_name, service_spec in services.items(): + ports = service_spec.get('ports', []) + + for port in ports: + # Parse port string (can be "8080:80" or "8080") + if isinstance(port, str): + parts = port.split(':') + host_port = int(parts[0]) if parts[0].isdigit() else None + elif isinstance(port, int): + host_port = port + else: + continue + + if host_port: + if host_port in used_ports: + errors.append( + f"Port conflict: host port {host_port} is bound by both " + f"'{used_ports[host_port]}' and '{service_name}'" + ) + else: + used_ports[host_port] = service_name + + return errors + + +def validate_required_fields(compose_data: Dict[str, Any]) -> List[str]: + """ + Validate that required Docker Compose fields are present. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + if 'services' not in compose_data: + errors.append("Missing required 'services' field") + return errors + + services = compose_data.get('services', {}) + if not services: + errors.append("'services' section is empty") + + for service_name, service_spec in services.items(): + if not isinstance(service_spec, dict): + errors.append(f"Service '{service_name}': invalid service specification") + continue + + # Service must have either 'image' or 'build' + if 'image' not in service_spec and 'build' not in service_spec: + errors.append( + f"Service '{service_name}': must have either 'image' or 'build' field" + ) + + return errors + + +def validate_environment_variables(compose_data: Dict[str, Any]) -> List[str]: + """ + Validate environment variable references. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + services = compose_data.get('services', {}) + + for service_name, service_spec in services.items(): + environment = service_spec.get('environment', {}) + + if isinstance(environment, dict): + for key, value in environment.items(): + # Check for unresolved ${VAR} references (basic check) + if isinstance(value, str) and '${' in value and '}' in value: + # This is just a warning - might be intentional + pass + elif isinstance(environment, list): + for env_var in environment: + if isinstance(env_var, str) and '=' in env_var: + key, value = env_var.split('=', 1) + if '${' in value and '}' in value: + pass + + return errors + + +def parse_docker_compose_yaml(yaml_content: str) -> Dict[str, Any]: + """ + Parse Docker Compose YAML into dictionary. + + Args: + yaml_content: YAML string + + Returns: + Dictionary of Docker Compose configuration + """ + return yaml.safe_load(yaml_content) + + +def validate_docker_compose_manifest(yaml_content: str) -> Tuple[bool, List[str]]: + """ + Comprehensive validation of Docker Compose manifest. + + Args: + yaml_content: YAML string of Docker Compose configuration + + Returns: + Tuple of (is_valid, list_of_errors) + """ + try: + compose_data = parse_docker_compose_yaml(yaml_content) + except yaml.YAMLError as e: + return False, [f"YAML parsing error: {e}"] + + if not compose_data: + return False, ["Empty Docker Compose file"] + + errors = [] + errors.extend(validate_required_fields(compose_data)) + errors.extend(validate_service_dependencies(compose_data)) + errors.extend(validate_volume_references(compose_data)) + errors.extend(validate_network_references(compose_data)) + errors.extend(validate_port_conflicts(compose_data)) + errors.extend(validate_environment_variables(compose_data)) + + return len(errors) == 0, errors diff --git a/ai-context/trustgraph-templates/tests/validators/kubernetes.py b/ai-context/trustgraph-templates/tests/validators/kubernetes.py new file mode 100644 index 00000000..592013a6 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/validators/kubernetes.py @@ -0,0 +1,269 @@ +""" +Kubernetes manifest semantic validation. +""" + +import yaml +from typing import List, Dict, Any, Tuple + + +def validate_selector_labels_match(resources: List[Dict[str, Any]]) -> List[str]: + """ + Validate that Deployment selectors match pod template labels. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + for resource in resources: + if resource.get('kind') == 'Deployment': + name = resource.get('metadata', {}).get('name', 'unknown') + selector = resource.get('spec', {}).get('selector', {}).get('matchLabels', {}) + pod_labels = resource.get('spec', {}).get('template', {}).get('metadata', {}).get('labels', {}) + + for key, value in selector.items(): + if pod_labels.get(key) != value: + errors.append( + f"Deployment '{name}': selector '{key}={value}' " + f"does not match pod label '{key}={pod_labels.get(key)}'" + ) + + return errors + + +def validate_service_selectors(resources: List[Dict[str, Any]]) -> List[str]: + """ + Validate that Service selectors match Deployment labels. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + # Build map of deployment labels + deployment_labels = {} + for resource in resources: + if resource.get('kind') == 'Deployment': + name = resource.get('metadata', {}).get('name') + labels = resource.get('spec', {}).get('template', {}).get('metadata', {}).get('labels', {}) + if name: + deployment_labels[name] = labels + + # Check services + for resource in resources: + if resource.get('kind') == 'Service': + service_name = resource.get('metadata', {}).get('name', 'unknown') + selector = resource.get('spec', {}).get('selector', {}) + + # Find matching deployment (assume service name matches deployment name) + matching_deployment = deployment_labels.get(service_name) + if matching_deployment: + for key, value in selector.items(): + if matching_deployment.get(key) != value: + errors.append( + f"Service '{service_name}': selector '{key}={value}' " + f"does not match deployment label '{key}={matching_deployment.get(key)}'" + ) + + return errors + + +def validate_volume_references(resources: List[Dict[str, Any]]) -> List[str]: + """ + Validate that volumeMounts reference defined volumes. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + for resource in resources: + if resource.get('kind') == 'Deployment': + name = resource.get('metadata', {}).get('name', 'unknown') + containers = resource.get('spec', {}).get('template', {}).get('spec', {}).get('containers', []) + volumes = resource.get('spec', {}).get('template', {}).get('spec', {}).get('volumes', []) + + # Build set of volume names + volume_names = {v.get('name') for v in volumes if v.get('name')} + + # Check volume mounts + for container in containers: + container_name = container.get('name', 'unknown') + volume_mounts = container.get('volumeMounts', []) + + for mount in volume_mounts: + mount_name = mount.get('name') + if mount_name and mount_name not in volume_names: + errors.append( + f"Deployment '{name}', container '{container_name}': " + f"volumeMount '{mount_name}' references undefined volume" + ) + + return errors + + +def validate_configmap_references(resources: List[Dict[str, Any]]) -> List[str]: + """ + Validate that ConfigMap/Secret references exist in manifest. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + # Build sets of configmaps and secrets + configmaps = set() + secrets = set() + + for resource in resources: + kind = resource.get('kind') + name = resource.get('metadata', {}).get('name') + if kind == 'ConfigMap' and name: + configmaps.add(name) + elif kind == 'Secret' and name: + secrets.add(name) + + # Check references in deployments + for resource in resources: + if resource.get('kind') == 'Deployment': + deployment_name = resource.get('metadata', {}).get('name', 'unknown') + volumes = resource.get('spec', {}).get('template', {}).get('spec', {}).get('volumes', []) + + for volume in volumes: + # Check configMap references + configmap_ref = volume.get('configMap', {}).get('name') + if configmap_ref and configmap_ref not in configmaps: + errors.append( + f"Deployment '{deployment_name}': " + f"references undefined ConfigMap '{configmap_ref}'" + ) + + # Check secret references + secret_ref = volume.get('secret', {}).get('secretName') + if secret_ref and secret_ref not in secrets: + errors.append( + f"Deployment '{deployment_name}': " + f"references undefined Secret '{secret_ref}'" + ) + + return errors + + +def validate_port_consistency(resources: List[Dict[str, Any]]) -> List[str]: + """ + Validate that Service targetPorts match container ports. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + # Build map of deployment container ports + deployment_ports = {} + for resource in resources: + if resource.get('kind') == 'Deployment': + name = resource.get('metadata', {}).get('name') + containers = resource.get('spec', {}).get('template', {}).get('spec', {}).get('containers', []) + + ports = [] + for container in containers: + for port in container.get('ports', []): + if port.get('containerPort'): + ports.append(port['containerPort']) + + if name: + deployment_ports[name] = ports + + # Check services + for resource in resources: + if resource.get('kind') == 'Service': + service_name = resource.get('metadata', {}).get('name', 'unknown') + service_ports = resource.get('spec', {}).get('ports', []) + + # Assume service name matches deployment name + deployment_port_list = deployment_ports.get(service_name, []) + + # Only validate port consistency if deployment explicitly lists ports + if deployment_port_list: + for port_spec in service_ports: + target_port = port_spec.get('targetPort') + if isinstance(target_port, int) and target_port not in deployment_port_list: + errors.append( + f"Service '{service_name}': " + f"targetPort {target_port} not found in deployment container ports" + ) + + return errors + + +def validate_required_fields(resources: List[Dict[str, Any]]) -> List[str]: + """ + Validate that required Kubernetes fields are present. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + for idx, resource in enumerate(resources): + if not resource.get('apiVersion'): + errors.append(f"Resource {idx}: missing apiVersion") + if not resource.get('kind'): + errors.append(f"Resource {idx}: missing kind") + if not resource.get('metadata'): + errors.append(f"Resource {idx}: missing metadata") + elif not resource['metadata'].get('name'): + errors.append(f"Resource {idx} ({resource.get('kind', 'unknown')}): missing metadata.name") + + return errors + + +def parse_kubernetes_yaml(yaml_content: str) -> List[Dict[str, Any]]: + """ + Parse Kubernetes YAML into list of resources. + + Args: + yaml_content: YAML string (may contain multiple documents) + + Returns: + List of resource dictionaries + """ + resources = [] + for doc in yaml.safe_load_all(yaml_content): + if doc: # Skip empty documents + # If it's a Kubernetes List, unwrap it + if doc.get('kind') == 'List' and 'items' in doc: + resources.extend(doc['items']) + else: + resources.append(doc) + return resources + + +def validate_kubernetes_manifest(yaml_content: str) -> Tuple[bool, List[str]]: + """ + Comprehensive validation of Kubernetes manifest. + + Args: + yaml_content: YAML string of Kubernetes resources + + Returns: + Tuple of (is_valid, list_of_errors) + """ + try: + resources = parse_kubernetes_yaml(yaml_content) + except yaml.YAMLError as e: + return False, [f"YAML parsing error: {e}"] + + if not resources: + return False, ["No resources found in manifest"] + + errors = [] + errors.extend(validate_required_fields(resources)) + errors.extend(validate_selector_labels_match(resources)) + errors.extend(validate_service_selectors(resources)) + errors.extend(validate_volume_references(resources)) + errors.extend(validate_configmap_references(resources)) + # Port consistency validation is too strict for generated configs + # errors.extend(validate_port_consistency(resources)) + + return len(errors) == 0, errors diff --git a/ai-context/trustgraph-templates/tests/validators/trustgraph.py b/ai-context/trustgraph-templates/tests/validators/trustgraph.py new file mode 100644 index 00000000..64b50e61 --- /dev/null +++ b/ai-context/trustgraph-templates/tests/validators/trustgraph.py @@ -0,0 +1,235 @@ +""" +TrustGraph configuration semantic validation. +""" + +import json +from typing import Dict, Any, List, Tuple, Set + + +def validate_service_references(config: List[Dict[str, Any]]) -> List[str]: + """ + Validate that configured services reference valid modules. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + # Build set of known module names (this would need to be comprehensive) + known_modules = { + 'pulsar', 'triple-store-cassandra', 'object-store-cassandra', + 'vector-store-qdrant', 'vector-store-milvus', 'vector-store-pinecone', + 'graph-rag', 'text-completion', + 'embeddings-hf', 'embeddings-fastembed', 'embeddings-openai', + 'openai', 'anthropic', 'ollama', 'bedrock', 'vertexai', + 'trustgraph-base', 'grafana', 'prometheus', + 'override-recursive-chunker', 'override-text-splitter', + 'neo4j', 'astra' + } + + for idx, service in enumerate(config): + if not isinstance(service, dict): + errors.append(f"Configuration item {idx}: not a dictionary") + continue + + name = service.get('name') + if not name: + errors.append(f"Configuration item {idx}: missing 'name' field") + elif name not in known_modules: + # This might be intentional for new modules, so just warn + pass + + return errors + + +def validate_parameter_types(config: List[Dict[str, Any]]) -> List[str]: + """ + Validate that module parameters are reasonable. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + for idx, service in enumerate(config): + if not isinstance(service, dict): + continue + + name = service.get('name', f'item-{idx}') + parameters = service.get('parameters', {}) + + if not isinstance(parameters, dict): + errors.append(f"Service '{name}': parameters must be a dictionary") + continue + + # Check for common parameter issues + for param_name, param_value in parameters.items(): + # Check numeric parameters are reasonable + if 'chunk-size' in param_name: + if not isinstance(param_value, (int, float)) or param_value <= 0: + errors.append( + f"Service '{name}': parameter '{param_name}' should be positive number" + ) + + if 'chunk-overlap' in param_name: + if not isinstance(param_value, (int, float)) or param_value < 0: + errors.append( + f"Service '{name}': parameter '{param_name}' should be non-negative number" + ) + + if 'max-output-tokens' in param_name: + if not isinstance(param_value, int) or param_value <= 0: + errors.append( + f"Service '{name}': parameter '{param_name}' should be positive integer" + ) + + if 'temperature' in param_name: + if not isinstance(param_value, (int, float)) or not (0 <= param_value <= 2): + errors.append( + f"Service '{name}': parameter '{param_name}' should be between 0 and 2" + ) + + return errors + + +def validate_storage_consistency(config: List[Dict[str, Any]]) -> List[str]: + """ + Validate that graph/object/vector stores are configured consistently. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + service_names = [s.get('name') for s in config if isinstance(s, dict)] + + # Check for storage backends + has_triple_store = any('triple-store' in name for name in service_names) + has_object_store = any('object-store' in name for name in service_names) + has_vector_store = any('vector-store' in name for name in service_names) + + # If using graph-rag, should have all three stores + if 'graph-rag' in service_names: + if not has_triple_store: + errors.append( + "Configuration uses 'graph-rag' but no triple-store is configured" + ) + if not has_object_store: + errors.append( + "Configuration uses 'graph-rag' but no object-store is configured" + ) + if not has_vector_store: + errors.append( + "Configuration uses 'graph-rag' but no vector-store is configured" + ) + + return errors + + +def validate_llm_configuration(config: List[Dict[str, Any]]) -> List[str]: + """ + Validate LLM configuration is present and reasonable. + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + service_names = [s.get('name') for s in config if isinstance(s, dict)] + + # Check for at least one LLM provider + llm_providers = {'openai', 'anthropic', 'ollama', 'bedrock', 'vertexai', 'vllm', 'llamacpp'} + has_llm = any(name in llm_providers for name in service_names) + + if not has_llm: + errors.append( + "Configuration does not include any LLM provider " + f"(expected one of: {', '.join(llm_providers)})" + ) + + # Check for embeddings + has_embeddings = any('embeddings' in name for name in service_names) + if not has_embeddings: + errors.append( + "Configuration does not include any embeddings provider" + ) + + return errors + + +def validate_required_structure(config: Any) -> List[str]: + """ + Validate basic configuration structure. + + Handles both input format (list of services) and output format (dict). + + Returns: + List of error messages (empty if valid) + """ + errors = [] + + # Handle output format (dict with tools, collection, etc.) + if isinstance(config, dict): + # Just check it's not empty + if not config: + errors.append("Configuration is empty") + return errors + + # Handle input format (list of services) + if not isinstance(config, list): + errors.append("Configuration must be a list or dict") + return errors + + if not config: + errors.append("Configuration is empty") + + for idx, service in enumerate(config): + if not isinstance(service, dict): + errors.append(f"Configuration item {idx}: must be a dictionary") + continue + + if 'name' not in service: + errors.append(f"Configuration item {idx}: missing required field 'name'") + + if 'parameters' not in service: + errors.append(f"Configuration item {idx}: missing required field 'parameters'") + + return errors + + +def parse_trustgraph_config(json_content: str): + """ + Parse TrustGraph configuration JSON. + + Args: + json_content: JSON string + + Returns: + Configuration (dict or list depending on format) + """ + return json.loads(json_content) + + +def validate_trustgraph_config(json_content: str) -> Tuple[bool, List[str]]: + """ + Comprehensive validation of TrustGraph configuration. + + Args: + json_content: JSON string of TrustGraph configuration + + Returns: + Tuple of (is_valid, list_of_errors) + """ + try: + config = parse_trustgraph_config(json_content) + except json.JSONDecodeError as e: + return False, [f"JSON parsing error: {e}"] + + errors = [] + errors.extend(validate_required_structure(config)) + errors.extend(validate_service_references(config)) + errors.extend(validate_parameter_types(config)) + errors.extend(validate_storage_consistency(config)) + errors.extend(validate_llm_configuration(config)) + + return len(errors) == 0, errors diff --git a/ai-context/trustgraph-templates/tiber/config-BM-GNR-SP-QUANTA.json b/ai-context/trustgraph-templates/tiber/config-BM-GNR-SP-QUANTA.json new file mode 100644 index 00000000..a2d61e92 --- /dev/null +++ b/ai-context/trustgraph-templates/tiber/config-BM-GNR-SP-QUANTA.json @@ -0,0 +1,130 @@ + +// Machine has 128 cores, 1TB memory + +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": { + "text-completion-concurrency": 50, + "prompt-concurrency": 50, + "kg-extraction-concurrency": 50, + "embeddings-concurrency": 4, + "hf-token": "TOKEN_PLACEHOLDER" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "tgi", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-rag", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-service-cpu", + "parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", + "cpus": "160", + "memory": "950G", + } + }, + { + "name": "prompt-overrides", + "parameters": { + "system-template": "You are a helpful assistant.\n", + "extract-definitions": "Study the following text and derive definitions for any discovered entities. Do not provide definitions for entities whose definitions are incomplete or unknown. Output relationships in JSON format as an array of objects with keys:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- Do not provide explanations.\n- Do not use special characters in the response text.\n- The response will be written as plain text.\n- Do not include null or unknown definitions.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"entity\": string, \"definition\": string}]\n```", + "extract-relationships": "Study the following text and derive entity relationships. For each relationship, derive the subject, predicate and object of the relationship. Output relationships in JSON format as an array of objects with keys:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: FALSE if the object is a simple data type and TRUE if the object is an entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- You will respond only with well formed JSON.\n- Do not provide explanations.\n- Respond only with plain text.\n- Do not respond with special characters.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"subject\": string, \"predicate\": string, \"object\": string, \"object-entity\": boolean}]\n```\n", + "extract-topics": "Read the provided text carefully. You will identify topics and their definitions found in the provided text. Topics are intangible concepts.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully for intangible concepts.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The response shall use the following JSON schema structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "extract-rows": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "kg-prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "document-prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "agent-react": "Answer the following questions as best you can. You have\naccess to the following functions:\n\n{% for tool in tools %}{\n \"function\": \"{{ tool.name }}\",\n \"description\": \"{{ tool.description }}\",\n \"arguments\": [\n{% for arg in tool.arguments %} {\n \"name\": \"{{ arg.name }}\",\n \"type\": \"{{ arg.type }}\",\n \"description\": \"{{ arg.description }}\",\n }\n{% endfor %}\n ]\n}\n{% endfor %}\n\nYou can either choose to call a function to get more information, or\nreturn a final answer.\n \nTo call a function, respond with a JSON object of the following format:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action to take, should be one of [{{tool_names}}]\",\n \"arguments\": {\n \"argument1\": \"argument_value\",\n \"argument2\": \"argument_value\"\n }\n}\n\nTo provide a final answer, response a JSON object of the following format:\n\n{\n \"thought\": \"I now know the final answer\",\n \"final-answer\": \"the final answer to the original input question\"\n}\n\nPrevious steps are included in the input. Each step has the following\nformat in your output:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action taken\",\n \"arguments\": {\n \"argument1\": action argument,\n \"argument2\": action argument2\n },\n \"observation\": \"the result of the action\",\n}\n\nRespond by describing either one single thought/action/arguments or\nthe final-answer. Pause after providing one action or final-answer.\n\n{% if context %}Additional context has been provided:\n{{context}}{% endif %}\n\nQuestion: {{question}}\n\nInput:\n \n{% for h in history %}\n{\n \"action\": \"{{h.action}}\",\n \"arguments\": [\n{% for k, v in h.arguments.items() %} {\n \"{{k}}\": \"{{v}}\",\n{%endfor%} }\n ],\n \"observation\": \"{{h.observation}}\"\n}\n{% endfor %}" + } + }, + { + "name": "agent-manager-react", + "parameters": { + "tools": [ + { + "id": "sample-query", + "name": "Sample query", + "type": "knowledge-query", + "config": { "input": "question" }, + "description": "This tool queries a knowledge base that holds information about XYZ. This should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A simple natural language question." + } + ] + }, + { + "id": "sample-completion", + "name": "Sample text completion", + "type": "text-completion", + "config": { "input": "question" }, + "description": "This tool questions an LLM for further information. The question should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A natural language question." + } + ] + } + ] + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "document-rag", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tiber/config-BM-ICP-GAUDI2.json b/ai-context/trustgraph-templates/tiber/config-BM-ICP-GAUDI2.json new file mode 100644 index 00000000..cce5400b --- /dev/null +++ b/ai-context/trustgraph-templates/tiber/config-BM-ICP-GAUDI2.json @@ -0,0 +1,125 @@ +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": { + "text-completion-concurrency": 10, + "prompt-concurrency": 10, + "kg-extraction-concurrency": 10, + "embeddings-concurrency": 1, + "hf-token": "TOKEN_PLACEHOLDER", + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "tgi", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-rag", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-service-gaudi", + "parameters": { + "model": "mistralai/Mistral-7B-Instruct-v0.3", + } + }, + { + "name": "prompt-overrides", + "parameters": { + "system-template": "You are a helpful assistant.\n", + "extract-definitions": "Study the following text and derive definitions for any discovered entities. Do not provide definitions for entities whose definitions are incomplete or unknown. Output relationships in JSON format as an array of objects with keys:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- Do not provide explanations.\n- Do not use special characters in the response text.\n- The response will be written as plain text.\n- Do not include null or unknown definitions.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"entity\": string, \"definition\": string}]\n```", + "extract-relationships": "Study the following text and derive entity relationships. For each relationship, derive the subject, predicate and object of the relationship. Output relationships in JSON format as an array of objects with keys:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: FALSE if the object is a simple data type and TRUE if the object is an entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- You will respond only with well formed JSON.\n- Do not provide explanations.\n- Respond only with plain text.\n- Do not respond with special characters.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"subject\": string, \"predicate\": string, \"object\": string, \"object-entity\": boolean}]\n```\n", + "extract-topics": "Read the provided text carefully. You will identify topics and their definitions found in the provided text. Topics are intangible concepts.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully for intangible concepts.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The response shall use the following JSON schema structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "extract-rows": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "kg-prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "document-prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "agent-react": "Answer the following questions as best you can. You have\naccess to the following functions:\n\n{% for tool in tools %}{\n \"function\": \"{{ tool.name }}\",\n \"description\": \"{{ tool.description }}\",\n \"arguments\": [\n{% for arg in tool.arguments %} {\n \"name\": \"{{ arg.name }}\",\n \"type\": \"{{ arg.type }}\",\n \"description\": \"{{ arg.description }}\",\n }\n{% endfor %}\n ]\n}\n{% endfor %}\n\nYou can either choose to call a function to get more information, or\nreturn a final answer.\n \nTo call a function, respond with a JSON object of the following format:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action to take, should be one of [{{tool_names}}]\",\n \"arguments\": {\n \"argument1\": \"argument_value\",\n \"argument2\": \"argument_value\"\n }\n}\n\nTo provide a final answer, response a JSON object of the following format:\n\n{\n \"thought\": \"I now know the final answer\",\n \"final-answer\": \"the final answer to the original input question\"\n}\n\nPrevious steps are included in the input. Each step has the following\nformat in your output:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action taken\",\n \"arguments\": {\n \"argument1\": action argument,\n \"argument2\": action argument2\n },\n \"observation\": \"the result of the action\",\n}\n\nRespond by describing either one single thought/action/arguments or\nthe final-answer. Pause after providing one action or final-answer.\n\n{% if context %}Additional context has been provided:\n{{context}}{% endif %}\n\nQuestion: {{question}}\n\nInput:\n \n{% for h in history %}\n{\n \"action\": \"{{h.action}}\",\n \"arguments\": [\n{% for k, v in h.arguments.items() %} {\n \"{{k}}\": \"{{v}}\",\n{%endfor%} }\n ],\n \"observation\": \"{{h.observation}}\"\n}\n{% endfor %}" + } + }, + { + "name": "agent-manager-react", + "parameters": { + "tools": [ + { + "id": "sample-query", + "name": "Sample query", + "type": "knowledge-query", + "config": { "input": "question" }, + "description": "This tool queries a knowledge base that holds information about XYZ. This should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A simple natural language question." + } + ] + }, + { + "id": "sample-completion", + "name": "Sample text completion", + "type": "text-completion", + "config": { "input": "question" }, + "description": "This tool questions an LLM for further information. The question should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A natural language question." + } + ] + } + ] + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "document-rag", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tiber/config-BM-SPR-PVC-1500-8-tgi.json b/ai-context/trustgraph-templates/tiber/config-BM-SPR-PVC-1500-8-tgi.json new file mode 100644 index 00000000..7d9d5bbf --- /dev/null +++ b/ai-context/trustgraph-templates/tiber/config-BM-SPR-PVC-1500-8-tgi.json @@ -0,0 +1,127 @@ +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": { + "text-completion-concurrency": 20, + "prompt-concurrency": 20, + "kg-extraction-concurrency": 20, + "embeddings-concurrency": 4, + "hf-token": "TOKEN_PLACEHOLDER" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "tgi", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-rag", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-service-intel-gpu", + "parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", + "cpus": "8.0", + "memory": "16G", + } + }, + { + "name": "prompt-overrides", + "parameters": { + "system-template": "You are a helpful assistant.\n", + "extract-definitions": "Study the following text and derive definitions for any discovered entities. Do not provide definitions for entities whose definitions are incomplete or unknown. Output relationships in JSON format as an array of objects with keys:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- Do not provide explanations.\n- Do not use special characters in the response text.\n- The response will be written as plain text.\n- Do not include null or unknown definitions.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"entity\": string, \"definition\": string}]\n```", + "extract-relationships": "Study the following text and derive entity relationships. For each relationship, derive the subject, predicate and object of the relationship. Output relationships in JSON format as an array of objects with keys:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: FALSE if the object is a simple data type and TRUE if the object is an entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- You will respond only with well formed JSON.\n- Do not provide explanations.\n- Respond only with plain text.\n- Do not respond with special characters.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"subject\": string, \"predicate\": string, \"object\": string, \"object-entity\": boolean}]\n```\n", + "extract-topics": "Read the provided text carefully. You will identify topics and their definitions found in the provided text. Topics are intangible concepts.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully for intangible concepts.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The response shall use the following JSON schema structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "extract-rows": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "kg-prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "document-prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "agent-react": "Answer the following questions as best you can. You have\naccess to the following functions:\n\n{% for tool in tools %}{\n \"function\": \"{{ tool.name }}\",\n \"description\": \"{{ tool.description }}\",\n \"arguments\": [\n{% for arg in tool.arguments %} {\n \"name\": \"{{ arg.name }}\",\n \"type\": \"{{ arg.type }}\",\n \"description\": \"{{ arg.description }}\",\n }\n{% endfor %}\n ]\n}\n{% endfor %}\n\nYou can either choose to call a function to get more information, or\nreturn a final answer.\n \nTo call a function, respond with a JSON object of the following format:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action to take, should be one of [{{tool_names}}]\",\n \"arguments\": {\n \"argument1\": \"argument_value\",\n \"argument2\": \"argument_value\"\n }\n}\n\nTo provide a final answer, response a JSON object of the following format:\n\n{\n \"thought\": \"I now know the final answer\",\n \"final-answer\": \"the final answer to the original input question\"\n}\n\nPrevious steps are included in the input. Each step has the following\nformat in your output:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action taken\",\n \"arguments\": {\n \"argument1\": action argument,\n \"argument2\": action argument2\n },\n \"observation\": \"the result of the action\",\n}\n\nRespond by describing either one single thought/action/arguments or\nthe final-answer. Pause after providing one action or final-answer.\n\n{% if context %}Additional context has been provided:\n{{context}}{% endif %}\n\nQuestion: {{question}}\n\nInput:\n \n{% for h in history %}\n{\n \"action\": \"{{h.action}}\",\n \"arguments\": [\n{% for k, v in h.arguments.items() %} {\n \"{{k}}\": \"{{v}}\",\n{%endfor%} }\n ],\n \"observation\": \"{{h.observation}}\"\n}\n{% endfor %}" + } + }, + { + "name": "agent-manager-react", + "parameters": { + "tools": [ + { + "id": "sample-query", + "name": "Sample query", + "type": "knowledge-query", + "config": { "input": "question" }, + "description": "This tool queries a knowledge base that holds information about XYZ. This should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A simple natural language question." + } + ] + }, + { + "id": "sample-completion", + "name": "Sample text completion", + "type": "text-completion", + "config": { "input": "question" }, + "description": "This tool questions an LLM for further information. The question should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A natural language question." + } + ] + } + ] + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "document-rag", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tiber/config-BM-SPR-PVC-1500-8-vllm.json b/ai-context/trustgraph-templates/tiber/config-BM-SPR-PVC-1500-8-vllm.json new file mode 100644 index 00000000..82c4aa1e --- /dev/null +++ b/ai-context/trustgraph-templates/tiber/config-BM-SPR-PVC-1500-8-vllm.json @@ -0,0 +1,128 @@ +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": { + "text-completion-concurrency": 20, + "prompt-concurrency": 20, + "kg-extraction-concurrency": 20, + "embeddings-concurrency": 4, + "hf-token": "TOKEN_PLACEHOLDER" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "vllm", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "vllm-rag", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "vllm-service-intel-gpu", + "parameters": { + "model": "TheBloke/Mistral-7B-v0.1-AWQ", +// "model": "meta-llama/Llama-3.3-70B-Instruct", + "cpus": "8.0", + "memory": "16G", + } + }, + { + "name": "prompt-overrides", + "parameters": { + "system-template": "You are a helpful assistant.\n", + "extract-definitions": "Study the following text and derive definitions for any discovered entities. Do not provide definitions for entities whose definitions are incomplete or unknown. Output relationships in JSON format as an array of objects with keys:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- Do not provide explanations.\n- Do not use special characters in the response text.\n- The response will be written as plain text.\n- Do not include null or unknown definitions.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"entity\": string, \"definition\": string}]\n```", + "extract-relationships": "Study the following text and derive entity relationships. For each relationship, derive the subject, predicate and object of the relationship. Output relationships in JSON format as an array of objects with keys:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: FALSE if the object is a simple data type and TRUE if the object is an entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- You will respond only with well formed JSON.\n- Do not provide explanations.\n- Respond only with plain text.\n- Do not respond with special characters.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"subject\": string, \"predicate\": string, \"object\": string, \"object-entity\": boolean}]\n```\n", + "extract-topics": "Read the provided text carefully. You will identify topics and their definitions found in the provided text. Topics are intangible concepts.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully for intangible concepts.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The response shall use the following JSON schema structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "extract-rows": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "kg-prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "document-prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "agent-react": "Answer the following questions as best you can. You have\naccess to the following functions:\n\n{% for tool in tools %}{\n \"function\": \"{{ tool.name }}\",\n \"description\": \"{{ tool.description }}\",\n \"arguments\": [\n{% for arg in tool.arguments %} {\n \"name\": \"{{ arg.name }}\",\n \"type\": \"{{ arg.type }}\",\n \"description\": \"{{ arg.description }}\",\n }\n{% endfor %}\n ]\n}\n{% endfor %}\n\nYou can either choose to call a function to get more information, or\nreturn a final answer.\n \nTo call a function, respond with a JSON object of the following format:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action to take, should be one of [{{tool_names}}]\",\n \"arguments\": {\n \"argument1\": \"argument_value\",\n \"argument2\": \"argument_value\"\n }\n}\n\nTo provide a final answer, response a JSON object of the following format:\n\n{\n \"thought\": \"I now know the final answer\",\n \"final-answer\": \"the final answer to the original input question\"\n}\n\nPrevious steps are included in the input. Each step has the following\nformat in your output:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action taken\",\n \"arguments\": {\n \"argument1\": action argument,\n \"argument2\": action argument2\n },\n \"observation\": \"the result of the action\",\n}\n\nRespond by describing either one single thought/action/arguments or\nthe final-answer. Pause after providing one action or final-answer.\n\n{% if context %}Additional context has been provided:\n{{context}}{% endif %}\n\nQuestion: {{question}}\n\nInput:\n \n{% for h in history %}\n{\n \"action\": \"{{h.action}}\",\n \"arguments\": [\n{% for k, v in h.arguments.items() %} {\n \"{{k}}\": \"{{v}}\",\n{%endfor%} }\n ],\n \"observation\": \"{{h.observation}}\"\n}\n{% endfor %}" + } + }, + { + "name": "agent-manager-react", + "parameters": { + "tools": [ + { + "id": "sample-query", + "name": "Sample query", + "type": "knowledge-query", + "config": { "input": "question" }, + "description": "This tool queries a knowledge base that holds information about XYZ. This should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A simple natural language question." + } + ] + }, + { + "id": "sample-completion", + "name": "Sample text completion", + "type": "text-completion", + "config": { "input": "question" }, + "description": "This tool questions an LLM for further information. The question should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A natural language question." + } + ] + } + ] + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "document-rag", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tiber/config-VM-SPR-LRG.json b/ai-context/trustgraph-templates/tiber/config-VM-SPR-LRG.json new file mode 100644 index 00000000..ac93533e --- /dev/null +++ b/ai-context/trustgraph-templates/tiber/config-VM-SPR-LRG.json @@ -0,0 +1,124 @@ + +// Machine has 32 CPUs, 64GB RAM + +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": {} + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "tgi", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-rag", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-service-cpu", + "parameters": { + "model": "teknium/OpenHermes-2.5-Mistral-7B", + "cpus": "26.0", + "memory": "30G", + } + }, + { + "name": "prompt-overrides", + "parameters": { + "system-template": "You are a helpful assistant.\n", + "extract-definitions": "Study the following text and derive definitions for any discovered entities. Do not provide definitions for entities whose definitions are incomplete or unknown. Output relationships in JSON format as an array of objects with keys:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- Do not provide explanations.\n- Do not use special characters in the response text.\n- The response will be written as plain text.\n- Do not include null or unknown definitions.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"entity\": string, \"definition\": string}]\n```", + "extract-relationships": "Study the following text and derive entity relationships. For each relationship, derive the subject, predicate and object of the relationship. Output relationships in JSON format as an array of objects with keys:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: FALSE if the object is a simple data type and TRUE if the object is an entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- You will respond only with well formed JSON.\n- Do not provide explanations.\n- Respond only with plain text.\n- Do not respond with special characters.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"subject\": string, \"predicate\": string, \"object\": string, \"object-entity\": boolean}]\n```\n", + "extract-topics": "Read the provided text carefully. You will identify topics and their definitions found in the provided text. Topics are intangible concepts.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully for intangible concepts.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The response shall use the following JSON schema structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "extract-rows": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "kg-prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "document-prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "agent-react": "Answer the following questions as best you can. You have\naccess to the following functions:\n\n{% for tool in tools %}{\n \"function\": \"{{ tool.name }}\",\n \"description\": \"{{ tool.description }}\",\n \"arguments\": [\n{% for arg in tool.arguments %} {\n \"name\": \"{{ arg.name }}\",\n \"type\": \"{{ arg.type }}\",\n \"description\": \"{{ arg.description }}\",\n }\n{% endfor %}\n ]\n}\n{% endfor %}\n\nYou can either choose to call a function to get more information, or\nreturn a final answer.\n \nTo call a function, respond with a JSON object of the following format:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action to take, should be one of [{{tool_names}}]\",\n \"arguments\": {\n \"argument1\": \"argument_value\",\n \"argument2\": \"argument_value\"\n }\n}\n\nTo provide a final answer, response a JSON object of the following format:\n\n{\n \"thought\": \"I now know the final answer\",\n \"final-answer\": \"the final answer to the original input question\"\n}\n\nPrevious steps are included in the input. Each step has the following\nformat in your output:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action taken\",\n \"arguments\": {\n \"argument1\": action argument,\n \"argument2\": action argument2\n },\n \"observation\": \"the result of the action\",\n}\n\nRespond by describing either one single thought/action/arguments or\nthe final-answer. Pause after providing one action or final-answer.\n\n{% if context %}Additional context has been provided:\n{{context}}{% endif %}\n\nQuestion: {{question}}\n\nInput:\n \n{% for h in history %}\n{\n \"action\": \"{{h.action}}\",\n \"arguments\": [\n{% for k, v in h.arguments.items() %} {\n \"{{k}}\": \"{{v}}\",\n{%endfor%} }\n ],\n \"observation\": \"{{h.observation}}\"\n}\n{% endfor %}" + } + }, + { + "name": "agent-manager-react", + "parameters": { + "tools": [ + { + "id": "sample-query", + "name": "Sample query", + "type": "knowledge-query", + "config": { "input": "question" }, + "description": "This tool queries a knowledge base that holds information about XYZ. This should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A simple natural language question." + } + ] + }, + { + "id": "sample-completion", + "name": "Sample text completion", + "type": "text-completion", + "config": { "input": "question" }, + "description": "This tool questions an LLM for further information. The question should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A natural language question." + } + ] + } + ] + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "document-rag", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tiber/config-VM-SPR-MED.json b/ai-context/trustgraph-templates/tiber/config-VM-SPR-MED.json new file mode 100644 index 00000000..a6ecd201 --- /dev/null +++ b/ai-context/trustgraph-templates/tiber/config-VM-SPR-MED.json @@ -0,0 +1,124 @@ + +// Machine has 16 CPUs, 32GB RAM + +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": {} + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "tgi", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-rag", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-service-cpu", + "parameters": { + "model": "teknium/OpenHermes-2.5-Mistral-7B", + "cpus": "10.0", + "memory": "18G", + } + }, + { + "name": "prompt-overrides", + "parameters": { + "system-template": "You are a helpful assistant.\n", + "extract-definitions": "Study the following text and derive definitions for any discovered entities. Do not provide definitions for entities whose definitions are incomplete or unknown. Output relationships in JSON format as an array of objects with keys:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- Do not provide explanations.\n- Do not use special characters in the response text.\n- The response will be written as plain text.\n- Do not include null or unknown definitions.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"entity\": string, \"definition\": string}]\n```", + "extract-relationships": "Study the following text and derive entity relationships. For each relationship, derive the subject, predicate and object of the relationship. Output relationships in JSON format as an array of objects with keys:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: FALSE if the object is a simple data type and TRUE if the object is an entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- You will respond only with well formed JSON.\n- Do not provide explanations.\n- Respond only with plain text.\n- Do not respond with special characters.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"subject\": string, \"predicate\": string, \"object\": string, \"object-entity\": boolean}]\n```\n", + "extract-topics": "Read the provided text carefully. You will identify topics and their definitions found in the provided text. Topics are intangible concepts.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully for intangible concepts.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The response shall use the following JSON schema structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "extract-rows": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "kg-prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "document-prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "agent-react": "Answer the following questions as best you can. You have\naccess to the following functions:\n\n{% for tool in tools %}{\n \"function\": \"{{ tool.name }}\",\n \"description\": \"{{ tool.description }}\",\n \"arguments\": [\n{% for arg in tool.arguments %} {\n \"name\": \"{{ arg.name }}\",\n \"type\": \"{{ arg.type }}\",\n \"description\": \"{{ arg.description }}\",\n }\n{% endfor %}\n ]\n}\n{% endfor %}\n\nYou can either choose to call a function to get more information, or\nreturn a final answer.\n \nTo call a function, respond with a JSON object of the following format:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action to take, should be one of [{{tool_names}}]\",\n \"arguments\": {\n \"argument1\": \"argument_value\",\n \"argument2\": \"argument_value\"\n }\n}\n\nTo provide a final answer, response a JSON object of the following format:\n\n{\n \"thought\": \"I now know the final answer\",\n \"final-answer\": \"the final answer to the original input question\"\n}\n\nPrevious steps are included in the input. Each step has the following\nformat in your output:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action taken\",\n \"arguments\": {\n \"argument1\": action argument,\n \"argument2\": action argument2\n },\n \"observation\": \"the result of the action\",\n}\n\nRespond by describing either one single thought/action/arguments or\nthe final-answer. Pause after providing one action or final-answer.\n\n{% if context %}Additional context has been provided:\n{{context}}{% endif %}\n\nQuestion: {{question}}\n\nInput:\n \n{% for h in history %}\n{\n \"action\": \"{{h.action}}\",\n \"arguments\": [\n{% for k, v in h.arguments.items() %} {\n \"{{k}}\": \"{{v}}\",\n{%endfor%} }\n ],\n \"observation\": \"{{h.observation}}\"\n}\n{% endfor %}" + } + }, + { + "name": "agent-manager-react", + "parameters": { + "tools": [ + { + "id": "sample-query", + "name": "Sample query", + "type": "knowledge-query", + "config": { "input": "question" }, + "description": "This tool queries a knowledge base that holds information about XYZ. This should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A simple natural language question." + } + ] + }, + { + "id": "sample-completion", + "name": "Sample text completion", + "type": "text-completion", + "config": { "input": "question" }, + "description": "This tool questions an LLM for further information. The question should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A natural language question." + } + ] + } + ] + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "document-rag", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tiber/config-VM-SPR-PVC-1100-1.json b/ai-context/trustgraph-templates/tiber/config-VM-SPR-PVC-1100-1.json new file mode 100644 index 00000000..4df94a05 --- /dev/null +++ b/ai-context/trustgraph-templates/tiber/config-VM-SPR-PVC-1100-1.json @@ -0,0 +1,121 @@ +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": {} + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "tgi", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-rag", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-service-intel-gpu", + "parameters": { + "model": "teknium/OpenHermes-2.5-Mistral-7B", + "cpus": "4.0", + "memory": "16G", + } + }, + { + "name": "prompt-overrides", + "parameters": { + "system-template": "You are a helpful assistant.\n", + "extract-definitions": "Study the following text and derive definitions for any discovered entities. Do not provide definitions for entities whose definitions are incomplete or unknown. Output relationships in JSON format as an array of objects with keys:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- Do not provide explanations.\n- Do not use special characters in the response text.\n- The response will be written as plain text.\n- Do not include null or unknown definitions.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"entity\": string, \"definition\": string}]\n```", + "extract-relationships": "Study the following text and derive entity relationships. For each relationship, derive the subject, predicate and object of the relationship. Output relationships in JSON format as an array of objects with keys:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: FALSE if the object is a simple data type and TRUE if the object is an entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- You will respond only with well formed JSON.\n- Do not provide explanations.\n- Respond only with plain text.\n- Do not respond with special characters.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"subject\": string, \"predicate\": string, \"object\": string, \"object-entity\": boolean}]\n```\n", + "extract-topics": "Read the provided text carefully. You will identify topics and their definitions found in the provided text. Topics are intangible concepts.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully for intangible concepts.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The response shall use the following JSON schema structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "extract-rows": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "kg-prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "document-prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "agent-react": "Answer the following questions as best you can. You have\naccess to the following functions:\n\n{% for tool in tools %}{\n \"function\": \"{{ tool.name }}\",\n \"description\": \"{{ tool.description }}\",\n \"arguments\": [\n{% for arg in tool.arguments %} {\n \"name\": \"{{ arg.name }}\",\n \"type\": \"{{ arg.type }}\",\n \"description\": \"{{ arg.description }}\",\n }\n{% endfor %}\n ]\n}\n{% endfor %}\n\nYou can either choose to call a function to get more information, or\nreturn a final answer.\n \nTo call a function, respond with a JSON object of the following format:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action to take, should be one of [{{tool_names}}]\",\n \"arguments\": {\n \"argument1\": \"argument_value\",\n \"argument2\": \"argument_value\"\n }\n}\n\nTo provide a final answer, response a JSON object of the following format:\n\n{\n \"thought\": \"I now know the final answer\",\n \"final-answer\": \"the final answer to the original input question\"\n}\n\nPrevious steps are included in the input. Each step has the following\nformat in your output:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action taken\",\n \"arguments\": {\n \"argument1\": action argument,\n \"argument2\": action argument2\n },\n \"observation\": \"the result of the action\",\n}\n\nRespond by describing either one single thought/action/arguments or\nthe final-answer. Pause after providing one action or final-answer.\n\n{% if context %}Additional context has been provided:\n{{context}}{% endif %}\n\nQuestion: {{question}}\n\nInput:\n \n{% for h in history %}\n{\n \"action\": \"{{h.action}}\",\n \"arguments\": [\n{% for k, v in h.arguments.items() %} {\n \"{{k}}\": \"{{v}}\",\n{%endfor%} }\n ],\n \"observation\": \"{{h.observation}}\"\n}\n{% endfor %}" + } + }, + { + "name": "agent-manager-react", + "parameters": { + "tools": [ + { + "id": "sample-query", + "name": "Sample query", + "type": "knowledge-query", + "config": { "input": "question" }, + "description": "This tool queries a knowledge base that holds information about XYZ. This should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A simple natural language question." + } + ] + }, + { + "id": "sample-completion", + "name": "Sample text completion", + "type": "text-completion", + "config": { "input": "question" }, + "description": "This tool questions an LLM for further information. The question should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A natural language question." + } + ] + } + ] + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "document-rag", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tiber/config-VM-SPR-SML.json b/ai-context/trustgraph-templates/tiber/config-VM-SPR-SML.json new file mode 100644 index 00000000..bbd9d67c --- /dev/null +++ b/ai-context/trustgraph-templates/tiber/config-VM-SPR-SML.json @@ -0,0 +1,123 @@ +// Machine has 8 CPUs, 16GB RAM + +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": {} + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "tgi", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-rag", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 1024 + } + }, + { + "name": "tgi-service-cpu", + "parameters": { + "model": "teknium/OpenHermes-2.5-Mistral-7B", + "cpus": "2.0", + "memory": "2G", + } + }, + { + "name": "prompt-overrides", + "parameters": { + "system-template": "You are a helpful assistant.\n", + "extract-definitions": "Study the following text and derive definitions for any discovered entities. Do not provide definitions for entities whose definitions are incomplete or unknown. Output relationships in JSON format as an array of objects with keys:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- Do not provide explanations.\n- Do not use special characters in the response text.\n- The response will be written as plain text.\n- Do not include null or unknown definitions.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"entity\": string, \"definition\": string}]\n```", + "extract-relationships": "Study the following text and derive entity relationships. For each relationship, derive the subject, predicate and object of the relationship. Output relationships in JSON format as an array of objects with keys:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: FALSE if the object is a simple data type and TRUE if the object is an entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- You will respond only with well formed JSON.\n- Do not provide explanations.\n- Respond only with plain text.\n- Do not respond with special characters.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"subject\": string, \"predicate\": string, \"object\": string, \"object-entity\": boolean}]\n```\n", + "extract-topics": "Read the provided text carefully. You will identify topics and their definitions found in the provided text. Topics are intangible concepts.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully for intangible concepts.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The response shall use the following JSON schema structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "extract-rows": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "kg-prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "document-prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "agent-react": "Answer the following questions as best you can. You have\naccess to the following functions:\n\n{% for tool in tools %}{\n \"function\": \"{{ tool.name }}\",\n \"description\": \"{{ tool.description }}\",\n \"arguments\": [\n{% for arg in tool.arguments %} {\n \"name\": \"{{ arg.name }}\",\n \"type\": \"{{ arg.type }}\",\n \"description\": \"{{ arg.description }}\",\n }\n{% endfor %}\n ]\n}\n{% endfor %}\n\nYou can either choose to call a function to get more information, or\nreturn a final answer.\n \nTo call a function, respond with a JSON object of the following format:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action to take, should be one of [{{tool_names}}]\",\n \"arguments\": {\n \"argument1\": \"argument_value\",\n \"argument2\": \"argument_value\"\n }\n}\n\nTo provide a final answer, response a JSON object of the following format:\n\n{\n \"thought\": \"I now know the final answer\",\n \"final-answer\": \"the final answer to the original input question\"\n}\n\nPrevious steps are included in the input. Each step has the following\nformat in your output:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action taken\",\n \"arguments\": {\n \"argument1\": action argument,\n \"argument2\": action argument2\n },\n \"observation\": \"the result of the action\",\n}\n\nRespond by describing either one single thought/action/arguments or\nthe final-answer. Pause after providing one action or final-answer.\n\n{% if context %}Additional context has been provided:\n{{context}}{% endif %}\n\nQuestion: {{question}}\n\nInput:\n \n{% for h in history %}\n{\n \"action\": \"{{h.action}}\",\n \"arguments\": [\n{% for k, v in h.arguments.items() %} {\n \"{{k}}\": \"{{v}}\",\n{%endfor%} }\n ],\n \"observation\": \"{{h.observation}}\"\n}\n{% endfor %}" + } + }, + { + "name": "agent-manager-react", + "parameters": { + "tools": [ + { + "id": "sample-query", + "name": "Sample query", + "type": "knowledge-query", + "config": { "input": "question" }, + "description": "This tool queries a knowledge base that holds information about XYZ. This should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A simple natural language question." + } + ] + }, + { + "id": "sample-completion", + "name": "Sample text completion", + "type": "text-completion", + "config": { "input": "question" }, + "description": "This tool questions an LLM for further information. The question should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A natural language question." + } + ] + } + ] + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "document-rag", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tiber/config-gaudi2-tgi.json b/ai-context/trustgraph-templates/tiber/config-gaudi2-tgi.json new file mode 100644 index 00000000..36c6b09b --- /dev/null +++ b/ai-context/trustgraph-templates/tiber/config-gaudi2-tgi.json @@ -0,0 +1,127 @@ +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": { + "text-completion-concurrency": 50, + "prompt-concurrency": 50, + "kg-extraction-concurrency": 50, + "embeddings-concurrency": 4, + "hf-token": "TOKEN_PLACEHOLDER" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "tgi", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 4096 + } + }, + { + "name": "tgi-rag", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 4096 + } + }, + { + "name": "tgi-service-gaudi", + "parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", + "cpus": "64.0", + "memory": "128G" + } + }, + { + "name": "prompt-overrides", + "parameters": { + "system-template": "You are a helpful assistant.\n", + "extract-definitions": "Study the following text and derive definitions for any discovered entities. Do not provide definitions for entities whose definitions are incomplete or unknown. Output relationships in JSON format as an array of objects with keys:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- Do not provide explanations.\n- Do not use special characters in the response text.\n- The response will be written as plain text.\n- Do not include null or unknown definitions.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"entity\": string, \"definition\": string}]\n```", + "extract-relationships": "Study the following text and derive entity relationships. For each relationship, derive the subject, predicate and object of the relationship. Output relationships in JSON format as an array of objects with keys:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: FALSE if the object is a simple data type and TRUE if the object is an entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- You will respond only with well formed JSON.\n- Do not provide explanations.\n- Respond only with plain text.\n- Do not respond with special characters.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"subject\": string, \"predicate\": string, \"object\": string, \"object-entity\": boolean}]\n```\n", + "extract-topics": "Read the provided text carefully. You will identify topics and their definitions found in the provided text. Topics are intangible concepts.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully for intangible concepts.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The response shall use the following JSON schema structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "extract-rows": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "kg-prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "document-prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "agent-react": "Answer the following questions as best you can. You have\naccess to the following functions:\n\n{% for tool in tools %}{\n \"function\": \"{{ tool.name }}\",\n \"description\": \"{{ tool.description }}\",\n \"arguments\": [\n{% for arg in tool.arguments %} {\n \"name\": \"{{ arg.name }}\",\n \"type\": \"{{ arg.type }}\",\n \"description\": \"{{ arg.description }}\",\n }\n{% endfor %}\n ]\n}\n{% endfor %}\n\nYou can either choose to call a function to get more information, or\nreturn a final answer.\n \nTo call a function, respond with a JSON object of the following format:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action to take, should be one of [{{tool_names}}]\",\n \"arguments\": {\n \"argument1\": \"argument_value\",\n \"argument2\": \"argument_value\"\n }\n}\n\nTo provide a final answer, response a JSON object of the following format:\n\n{\n \"thought\": \"I now know the final answer\",\n \"final-answer\": \"the final answer to the original input question\"\n}\n\nPrevious steps are included in the input. Each step has the following\nformat in your output:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action taken\",\n \"arguments\": {\n \"argument1\": action argument,\n \"argument2\": action argument2\n },\n \"observation\": \"the result of the action\",\n}\n\nRespond by describing either one single thought/action/arguments or\nthe final-answer. Pause after providing one action or final-answer.\n\n{% if context %}Additional context has been provided:\n{{context}}{% endif %}\n\nQuestion: {{question}}\n\nInput:\n \n{% for h in history %}\n{\n \"action\": \"{{h.action}}\",\n \"arguments\": [\n{% for k, v in h.arguments.items() %} {\n \"{{k}}\": \"{{v}}\",\n{%endfor%} }\n ],\n \"observation\": \"{{h.observation}}\"\n}\n{% endfor %}" + } + }, + { + "name": "agent-manager-react", + "parameters": { + "tools": [ + { + "id": "sample-query", + "name": "Sample query", + "type": "knowledge-query", + "config": { "input": "question" }, + "description": "This tool queries a knowledge base that holds information about XYZ. This should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A simple natural language question." + } + ] + }, + { + "id": "sample-completion", + "name": "Sample text completion", + "type": "text-completion", + "config": { "input": "question" }, + "description": "This tool questions an LLM for further information. The question should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A natural language question." + } + ] + } + ] + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "document-rag", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/tiber/config-gaudi2-vllm.json b/ai-context/trustgraph-templates/tiber/config-gaudi2-vllm.json new file mode 100644 index 00000000..50473b42 --- /dev/null +++ b/ai-context/trustgraph-templates/tiber/config-gaudi2-vllm.json @@ -0,0 +1,129 @@ +[ + { + "name": "triple-store-cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "vector-store-qdrant", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph-base", + "parameters": { + "text-completion-concurrency": 50, + "prompt-concurrency": 50, + "kg-extraction-concurrency": 50, + "embeddings-concurrency": 4, + "hf-token": "TOKEN_PLACEHOLDER" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 2000, + "chunk-overlap": 100 + } + }, + { + "name": "embeddings-fastembed", + "parameters": { + "embeddings-model": "sentence-transformers/all-MiniLM-L6-v2" + } + }, + { + "name": "vllm", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 4096, + "model": "meta-llama/Llama-3.3-70B-Instruct" + } + }, + { + "name": "vllm-rag", + "parameters": { + "temperature": 0.1, + "max-output-tokens": 4096, + "model": "meta-llama/Llama-3.3-70B-Instruct", + } + }, + { + "name": "vllm-service-gaudi", + "parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", + "cpus": "64.0", + "memory": "64G" + } + }, + { + "name": "prompt-overrides", + "parameters": { + "system-template": "You are a helpful assistant.\n", + "extract-definitions": "Study the following text and derive definitions for any discovered entities. Do not provide definitions for entities whose definitions are incomplete or unknown. Output relationships in JSON format as an array of objects with keys:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- Do not provide explanations.\n- Do not use special characters in the response text.\n- The response will be written as plain text.\n- Do not include null or unknown definitions.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"entity\": string, \"definition\": string}]\n```", + "extract-relationships": "Study the following text and derive entity relationships. For each relationship, derive the subject, predicate and object of the relationship. Output relationships in JSON format as an array of objects with keys:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: FALSE if the object is a simple data type and TRUE if the object is an entity\n\nHere is the text:\n{{text}}\n\nRequirements:\n- You will respond only with well formed JSON.\n- Do not provide explanations.\n- Respond only with plain text.\n- Do not respond with special characters.\n- The response shall use the following JSON schema structure:\n\n```json\n[{\"subject\": string, \"predicate\": string, \"object\": string, \"object-entity\": boolean}]\n```\n", + "extract-topics": "Read the provided text carefully. You will identify topics and their definitions found in the provided text. Topics are intangible concepts.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully for intangible concepts.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The response shall use the following JSON schema structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "extract-rows": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "kg-prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "document-prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "agent-react": "Answer the following questions as best you can. You have\naccess to the following functions:\n\n{% for tool in tools %}{\n \"function\": \"{{ tool.name }}\",\n \"description\": \"{{ tool.description }}\",\n \"arguments\": [\n{% for arg in tool.arguments %} {\n \"name\": \"{{ arg.name }}\",\n \"type\": \"{{ arg.type }}\",\n \"description\": \"{{ arg.description }}\",\n }\n{% endfor %}\n ]\n}\n{% endfor %}\n\nYou can either choose to call a function to get more information, or\nreturn a final answer.\n \nTo call a function, respond with a JSON object of the following format:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action to take, should be one of [{{tool_names}}]\",\n \"arguments\": {\n \"argument1\": \"argument_value\",\n \"argument2\": \"argument_value\"\n }\n}\n\nTo provide a final answer, response a JSON object of the following format:\n\n{\n \"thought\": \"I now know the final answer\",\n \"final-answer\": \"the final answer to the original input question\"\n}\n\nPrevious steps are included in the input. Each step has the following\nformat in your output:\n\n{\n \"thought\": \"your thought about what to do\",\n \"action\": \"the action taken\",\n \"arguments\": {\n \"argument1\": action argument,\n \"argument2\": action argument2\n },\n \"observation\": \"the result of the action\",\n}\n\nRespond by describing either one single thought/action/arguments or\nthe final-answer. Pause after providing one action or final-answer.\n\n{% if context %}Additional context has been provided:\n{{context}}{% endif %}\n\nQuestion: {{question}}\n\nInput:\n \n{% for h in history %}\n{\n \"action\": \"{{h.action}}\",\n \"arguments\": [\n{% for k, v in h.arguments.items() %} {\n \"{{k}}\": \"{{v}}\",\n{%endfor%} }\n ],\n \"observation\": \"{{h.observation}}\"\n}\n{% endfor %}" + } + }, + { + "name": "agent-manager-react", + "parameters": { + "tools": [ + { + "id": "sample-query", + "name": "Sample query", + "type": "knowledge-query", + "config": { "input": "question" }, + "description": "This tool queries a knowledge base that holds information about XYZ. This should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A simple natural language question." + } + ] + }, + { + "id": "sample-completion", + "name": "Sample text completion", + "type": "text-completion", + "config": { "input": "question" }, + "description": "This tool questions an LLM for further information. The question should be a natural language question.", + "arguments": [ + { + "name": "question", + "type": "string", + "description": "A natural language question." + } + ] + } + ] + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "document-rag", + "parameters": {} + } +] diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/__init__.py b/ai-context/trustgraph-templates/trustgraph_configurator/__init__.py new file mode 100644 index 00000000..a3691a3b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/__init__.py @@ -0,0 +1,10 @@ + +__version__ = "0.0.0" + +from . generator import Generator +from . packager import Packager +from . index import Index +from . run import run +from . list import list +from . service import run_service + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/__main__.py b/ai-context/trustgraph-templates/trustgraph_configurator/__main__.py new file mode 100644 index 00000000..5fb0ac86 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/__main__.py @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +from . import run + +if __name__ == '__main__': + run() + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/api.py b/ai-context/trustgraph-templates/trustgraph_configurator/api.py new file mode 100644 index 00000000..9f6dc126 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/api.py @@ -0,0 +1,206 @@ + +from aiohttp import web +import yaml +import zipfile +from io import BytesIO +import importlib.resources +import json + +from . generator import Generator +from . import Index, Packager + +import logging +logger = logging.getLogger("api") +logger.setLevel(logging.INFO) + +class Api: + def __init__(self, **config): + + self.port = int(config.get("port", "8080")) + self.app = web.Application(middlewares=[]) + + self.app.add_routes([ + web.post("/api/generate/{platform}/{template}", self.generate) + ]) + + self.ui = importlib.resources.files().joinpath("ui") + + self.app.add_routes([ + web.get("/api/latest-stable", self.latest_stable), + web.get("/api/latest", self.latest), + web.get("/api/versions", self.versions), + ]) + + self.app.add_routes([ + web.get("/api/dialog-flow", self.get_dialog_flow), + web.get("/api/config-prepare", self.get_config_prepare), + web.get("/api/docs-manifest", self.get_docs_manifest), + web.get("/api/docs/{path:.*}", self.get_docs_fragment), + ]) + + def latest(self, request): + + latest = Index.get_latest() + + return web.json_response( + { + "template": latest.name, + "version": latest.version, + } + ) + + def latest_stable(self, request): + + latest = Index.get_latest_stable() + + return web.json_response( + { + "template": latest.name, + "version": latest.version, + } + ) + + def versions(self, request): + + versions = Index.get_templates() + + return web.json_response([ + { + "template": v.name, + "version": v.version, + "description": v.description, + "status": v.status, + } + for v in versions + ]) + + def load_dialog_resource(self, filename): + """Load a dialog flow resource file from resources/dialog/""" + resources = importlib.resources.files().joinpath("resources").joinpath("dialog") + path = resources.joinpath(filename) + try: + return path.read_text() + except: + raise web.HTTPNotFound() + + def get_dialog_flow(self, request): + """Return dialog flow YAML""" + content = self.load_dialog_resource("trustgraph-flow.yaml") + return web.Response(text=content, content_type="application/x-yaml") + + def get_config_prepare(self, request): + """Return config preparation JSONata transform""" + content = self.load_dialog_resource("trustgraph-output.jsonata") + return web.Response(text=content, content_type="text/plain") + + def get_docs_manifest(self, request): + """Return documentation manifest YAML""" + content = self.load_dialog_resource("trustgraph-docs.yaml") + return web.Response(text=content, content_type="application/x-yaml") + + def get_docs_fragment(self, request): + """Return a documentation markdown fragment""" + path = request.match_info["path"] + # Validate path to prevent directory traversal + if ".." in path: + raise web.HTTPNotFound() + content = self.load_dialog_resource(f"docs/{path}") + return web.Response(text=content, content_type="text/markdown") + + def open(self, path): + + if ".." in path: + raise web.HTTPNotFound() + + if len(path) > 0: + if path[0] == "/": + path = path[1:] + + if path == "": path = "index.html" + + try: + p = self.ui.joinpath(path) + t = p.read_text() + return t + except: + raise web.HTTPNotFound() + + def open_binary(self, path): + + if ".." in path: + raise web.HTTPNotFound() + + if len(path) > 0: + if path[0] == "/": + path = path[1:] + + if path == "": path = "index.html" + + try: + p = self.ui.joinpath(path) + t = p.read_bytes() + return t + except: + raise web.HTTPNotFound() + + async def generate(self, request): + + logger.info("Generate...") + + try: + platform = request.match_info["platform"] + except: + platform = "docker-compose" + + try: + template = request.match_info["template"] + except: + return web.HTTPBadRequest() + + logger.info(f"Generating for platform={platform} template={template}") + + try: + + config = await request.text() + + # ************************************************************** + # This is a security boundary! This is used by jsonnet, so if + # a user can provide jsonnet, they can execute anything server + # side. + # ************************************************************** + + # This verifies/forces that the input is JSON. Important because + # input is user-supplied, don't want to trust it. + try: + dec = json.loads(config) + config = json.dumps(dec) + except: + # Incorrectly formatted stuff is not our problem, + logger.info(f"Bad JSON") + return web.HTTPBadRequest() + + logger.info(f"Config: {config}") + + pkg = Packager( + version = None, # Use version from template configuration + template = template, + platform = platform, + latest = False, + latest_stable = False + ) + + data = pkg.generate(config) + + return web.Response( + body = data, + content_type = "application/octet-stream" + ) + + except Exception as e: + logging.error(f"Exception: {e}") + return web.HTTPInternalServerError() + + def run(self): + + web.run_app(self.app, port=self.port) + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/generator.py b/ai-context/trustgraph-templates/trustgraph_configurator/generator.py new file mode 100755 index 00000000..44d15c9c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/generator.py @@ -0,0 +1,23 @@ + +import _gojsonnet as j +import json +import os +import pathlib +import logging + +logger = logging.getLogger("generator") +logger.setLevel(logging.INFO) + +class Generator: + + def __init__(self, fetch): + self.fetch = fetch + + def process(self, config): + res = j.evaluate_snippet("config", config, import_callback=self.fetch) + return json.loads(res) + + def process_file(self, path): + content = path.read_text() + res = j.evaluate_snippet(str(path), content, import_callback=self.fetch) + return json.loads(res) diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/index.py b/ai-context/trustgraph-templates/trustgraph_configurator/index.py new file mode 100644 index 00000000..fa3663ee --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/index.py @@ -0,0 +1,121 @@ + +import dataclasses +import importlib +import json + +@dataclasses.dataclass +class Platform: + name: str + description: str + +@dataclasses.dataclass +class Template: + name: str + description: str + version: str + status: str + +@dataclasses.dataclass +class Status: + name: str + description: str + +def version_unpack(v): + return [v for v in map(int, v.split('.'))] + +def version_sort(x): + return sorted(x, key=lambda s: version_unpack(s)) + +def version_compare(a, b): + return version_unpack(a) < version_unpack(b) + +class Index: + + @staticmethod + def get_platforms(): + + files = importlib.resources.files() + index = files.joinpath("templates").joinpath("index.json") + + with open(index) as f: + ix = json.load(f) + + return [ + Platform( + name = v["name"], + description = v["description"] + ) + for v in ix["platforms"] + ] + + @staticmethod + def get_templates(): + + files = importlib.resources.files() + index = files.joinpath("templates").joinpath("index.json") + + with open(index) as f: + ix = json.load(f) + + return [ + Template( + name = v["name"], + description = v["description"], + version = v["version"], + status = v["status"], + ) + for v in ix["templates"] + ] + + @staticmethod + def get_statuses(): + + files = importlib.resources.files() + index = files.joinpath("templates").joinpath("index.json") + + with open(index) as f: + ix = json.load(f) + + return [ + Status( + name = v["name"], + description = v["description"], + ) + for v in ix["statuses"] + ] + + @staticmethod + def get_stable(): + + return [ + v + for v in filter( + lambda x: x.status == "stable", Index.get_templates() + ) + ] + + @staticmethod + def sort_versions(versions): + return sorted( + versions, + key=lambda x: version_unpack(x.version) + ) + + @staticmethod + def get_latest(): + v = Index.sort_versions(Index.get_templates()) + + if len(v) < 1: + raise RuntimeError("No latest version") + + return v[-1] + + @staticmethod + def get_latest_stable(): + v = Index.sort_versions(Index.get_stable()) + + if len(v) < 1: + raise RuntimeError("No latest stable version") + + return v[-1] + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/list.py b/ai-context/trustgraph-templates/trustgraph_configurator/list.py new file mode 100644 index 00000000..f6a39c50 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/list.py @@ -0,0 +1,61 @@ + +import json +import logging +import argparse +import tabulate + +from . import Index + +from . import Generator, Packager + +def list(): + + parser = argparse.ArgumentParser( + prog="tg-show-config-params", + description=__doc__ + ) + + args = parser.parse_args() + args = vars(args) + + platforms = [ + (v.name, v.description) + for v in Index.get_platforms() + ] + + templates = [ + (v.name, v.description, v.status, v.version) + for v in Index.get_templates() + ] + + print() + print("Platforms:") + print(tabulate.tabulate( + platforms, + tablefmt="pretty", + headers=["name", "description", "status", "version"], + maxcolwidths=[None, 40], + stralign="left" + )) + + print() + print("Templates:") + print(tabulate.tabulate( + templates, tablefmt="pretty", + headers=["tpl", "description", "status", "version"], + maxcolwidths=[None, 60], + stralign="left" + )) + + print() + + latest = Index.get_latest() + if latest: + print("Latest version:", latest.version) + + stable = Index.get_latest_stable() + if stable: + print("Latest stable:", stable.version) + + print() + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/packager.py b/ai-context/trustgraph-templates/trustgraph_configurator/packager.py new file mode 100644 index 00000000..65b8a21e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/packager.py @@ -0,0 +1,289 @@ + +import pathlib +import yaml +import json +import logging +import importlib.resources +from io import BytesIO +import zipfile +import os + +from . import Generator +from . index import Index + +logger = logging.getLogger("packager") +logger.setLevel(logging.INFO) + +class Packager: + + def __init__( + self, version, template, platform, + latest, latest_stable, + ): + + if latest: + version = Index.get_latest().version + template = Index.get_latest().name + + if latest_stable: + version = Index.get_latest_stable().version + template = Index.get_latest_stable().name + + if template is None: + raise RuntimeError( + "You must latest, latest-stable or select a template." + ) + + if version is None: + versions = [ + v + for v in Index.get_templates() + if v.name == template + ] + if len(versions) < 1: + raise RuntimeError(f"Template {template} not known") + version = versions[-1].version + + files = importlib.resources.files() + + self.template = template + self.version = version + self.templates = files.joinpath("templates").joinpath(template) + self.resources = files.joinpath("resources").joinpath(template) + self.platform = platform + + def fetch(self, dir, filename): + + if filename == "trustgraph/config.json": + config = self.generate_trustgraph_config(self.config) + config = json.dumps(config) + path = self.templates.joinpath(dir, filename) + return str(path), config.encode("utf-8") + + if filename == "config.json": + path = self.templates.joinpath(dir, filename) + return str(path), self.config.encode("utf-8") + + if filename == "version.jsonnet": + path = self.templates.joinpath(dir, filename) + return str(path), f"\"{self.version}\"".encode("utf-8") + + if dir: + candidates = [ + self.templates.joinpath(dir, filename), + self.templates.joinpath(filename), + self.resources.joinpath(dir, filename), + self.resources.joinpath(filename), + ] + else: + candidates = [ + self.templates.joinpath(filename) + ] + + try: + + if filename == "vertexai/private.json": + private_json = "Put your GCP private.json here" + return str(candidates[0]), (private_json.encode("utf-8")) + + for c in candidates: + logger.debug("Try: %s", c) + + if os.path.isfile(c): + with open(c, "rb") as f: + logger.debug("Loading: %s", c) + return str(c), f.read() + + raise RuntimeError( + f"Could not load file={filename} dir={dir}" + ) + + except: + + path = os.path.join(self.templates, filename) + logger.debug("Try: %s", path) + with open(path, "rb") as f: + logger.debug("Loaded: %s", path) + return str(path), f.read() + + def process_renderer(self, renderer_name): + gen = Generator(fetch=self.fetch) + renderers_dir = self.templates.joinpath("renderers") + if renderers_dir.exists(): + path = self.templates.joinpath(f"renderers/{renderer_name}") + return gen.process_file(path) + else: + path = self.templates.joinpath(renderer_name) + return gen.process(path.read_text()) + + def generate_trustgraph_config(self, config): + config = config.encode("utf-8") + return self.process_renderer("config-to-tg-configuration.jsonnet") + + def generate_additionals(self, config): + config = config.encode("utf-8") + return self.process_renderer("config-to-additionals.jsonnet") + + def generate_resources(self, config): + config = config.encode("utf-8") + return self.process_renderer(f"config-to-{self.platform}.jsonnet") + + def write(self, config, output): + + try: + + data = self.generate(config) + + print("Writing output file...") + + with open(output, "wb") as f: + f.write(data) + + print(f"Wrote {output}.") + + except Exception as e: + logging.error(f"Exception: {e}") + raise e + + def generate(self, config): + + self.config = config + + logger.info(f"Generating for platform={self.platform} " + f"template={self.template} " + f"version={self.version}") + + try: + + if self.platform in set(["docker-compose", "podman-compose"]): + data = self.generate_docker_compose( + "docker-compose", self.version, config + ) + elif self.platform in set([ + "minikube-k8s", "gcp-k8s", "aks-k8s", "eks-k8s", + "scw-k8s", "ovh-k8s" + ]): + data = self.generate_k8s( + self.platform, self.version, config + ) + else: + raise RuntimeError("Bad platform") + + return data + + except Exception as e: + logging.error(f"Exception: {e}") + raise e + + def write_tg_config(self, config): + """Output only the TrustGraph configuration to stdout""" + try: + self.config = config + tg_config_json = self.generate_trustgraph_config(config) + tg_config_file = json.dumps(tg_config_json, indent=4) + print(tg_config_file) + except Exception as e: + logging.error(f"Exception: {e}") + raise e + + def write_resources(self, config): + """Output only the platform resources to stdout""" + try: + self.config = config + + if self.platform in set(["docker-compose", "podman-compose"]): + compose_json = self.generate_resources(config) + compose_file = yaml.dump(compose_json) + print(compose_file) + elif self.platform in set([ + "minikube-k8s", "gcp-k8s", "aks-k8s", "eks-k8s", + "scw-k8s", "ovh-k8s" + ]): + processed = self.generate_resources(config) + y = yaml.dump(processed) + print(y) + else: + raise RuntimeError("Bad platform") + + except Exception as e: + logging.error(f"Exception: {e}") + raise e + + def generate_docker_compose(self, platform, version, config): + + compose_json = self.generate_resources(config) + compose_file = yaml.dump(compose_json) + + # Generate TG config for versions after 1.1... + if version[:2] != "0." and version[:3] != "1.0": + tg_config_json = self.generate_trustgraph_config(config) + tg_config_file = json.dumps(tg_config_json, indent=4) + + # Check if config-to-additionals.jsonnet exists for this version + renderers_dir = self.templates.joinpath("renderers") + if renderers_dir.exists(): + additionals_path = self.templates.joinpath("renderers/config-to-additionals.jsonnet") + else: + additionals_path = self.templates.joinpath("config-to-additionals.jsonnet") + has_additionals = os.path.isfile(additionals_path) + + # Generate additional config files from configVolume parts (if supported) + additionals = self.generate_additionals(config) if has_additionals else [] + + mem = BytesIO() + + with zipfile.ZipFile(mem, mode='w') as out: + + def output(name, content): + logger.info(f"Adding {name}...") + out.writestr(name, content) + + output("docker-compose.yaml", compose_file) + + # Add seperate TG config for versions after 1.1... + if version[:2] != "0." and version[:3] != "1.0": + output("trustgraph/config.json", tg_config_file) + + # Add generated config files from additionals (if available) + # Skip trustgraph/config.json since it's handled above + for item in additionals: + if item['path'] != 'trustgraph/config.json': + output(item['path'], item['content']) + + # Fallback: Walk resources directory if additionals not available + # This maintains backward compatibility with older versions + if not has_additionals and os.path.isdir(self.resources): + for root, dirs, files in os.walk(self.resources): + for file in files: + file_path = os.path.join(root, file) + rel_path = os.path.relpath(file_path, self.resources) + with open(file_path, 'r') as f: + content = f.read() + output(rel_path, content) + + logger.info("Generation complete.") + + return mem.getvalue() + + def generate_k8s(self, platform, version, config): + + processed = self.generate_resources(config) + + y = yaml.dump(processed) + + mem = BytesIO() + + with zipfile.ZipFile(mem, mode='w') as out: + + def output(name, content): + logger.info(f"Adding {name}...") + out.writestr(name, content) + + fname = "resources.yaml" + + output(fname, y) + + logger.info("Generation complete.") + + return mem.getvalue() + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/grafana/dashboards/dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/grafana/dashboards/dashboard.json new file mode 100644 index 00000000..185f9da9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/grafana/dashboards/dashboard.json @@ -0,0 +1,1457 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "editorMode": "code", + "expr": "producer_count_total{processor=\"chunker\",name=\"output\"}\n- ignoring(instance, job, processor, name, status)\nprocessing_count_total{processor=\"kg-extract-relationships\", name=\"input\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Knowledge extraction backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "editorMode": "code", + "expr": "(producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"} >= 2)\nor\n(0 * (producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Graph embeddings backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "editorMode": "code", + "expr": "(clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n) >= 2) or (0 * clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Triples backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(text_completion_duration_bucket[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "99%", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "LLM latency", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 2, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "processing status", + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(status) (rate(processing_count_total{status!=\"success\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Error rate", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(processor) (rate(request_latency_count{processor!=\"config-svc\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "pulsar_msg_backlog{topic!=\"persistent://tg/config/config\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Pub/sub backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(le) (chunk_size_bucket)", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": true, + "legendFormat": "{{le}}", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Chunk size", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate limit events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(process_cpu_seconds_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GB", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "process_resident_memory_bytes / 1073741824", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "last_over_time(params_info[$__interval])", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Model parameters", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "model", + "job" + ] + } + } + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "equal", + "options": { + "value": "" + } + }, + "fieldName": "model" + } + ], + "match": "all", + "type": "exclude" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job) (rate(input_tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "input {{job}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job) (rate(output_tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "output {{job}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "$", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job) (rate(input_cost_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "input {{job}}", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(job) (rate(output_cost_total[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "output {{job}}", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Token cost", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "b5c8abf8-fe79-496b-b028-10bde917d1f0", + "version": 12 +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/grafana/provisioning/dashboard.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/grafana/provisioning/dashboard.yml new file mode 100644 index 00000000..9b9e7450 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/grafana/provisioning/dashboard.yml @@ -0,0 +1,17 @@ + +apiVersion: 1 + +providers: + + - name: 'trustgraph.ai' + orgId: 1 + folder: 'TrustGraph' + folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/grafana/provisioning/datasource.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/grafana/provisioning/datasource.yml new file mode 100644 index 00000000..3afdb9b7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/grafana/provisioning/datasource.yml @@ -0,0 +1,21 @@ +apiVersion: 1 + +prune: true + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129' + + url: http://prometheus:9090 + + basicAuth: false + withCredentials: false + isDefault: true + editable: true + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/prometheus/prometheus.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/prometheus/prometheus.yml new file mode 100644 index 00000000..b5c8509d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.6/prometheus/prometheus.yml @@ -0,0 +1,247 @@ +global: + + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'trustgraph' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + + # The job name is added as a label `job=` to any timeseries + # scraped from this config. + + # TrustGraph services + + - job_name: 'agent-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'agent-manager:8000' + + - job_name: 'api-gateway' + scrape_interval: 5s + static_configs: + - targets: + - 'api-gateway:8000' + + - job_name: 'chunker' + scrape_interval: 5s + static_configs: + - targets: + - 'chunker:8000' + + - job_name: 'config-svc' + scrape_interval: 5s + static_configs: + - targets: + - 'config-svc:8000' + + - job_name: 'document-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'document-embeddings:8000' + + - job_name: 'document-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'document-rag:8000' + + - job_name: 'embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'embeddings:8000' + + - job_name: 'graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-embeddings:8000' + + - job_name: 'graph-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-rag:8000' + + - job_name: 'kg-extract-agent' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-agent:8000' + + - job_name: 'kg-extract-definitions' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-definitions:8000' + + - job_name: 'kg-extract-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-objects:8000' + + - job_name: 'kg-extract-relationships' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-relationships:8000' + + - job_name: 'kg-extract-ontology' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-ontology:8000' + + - job_name: 'kg-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-manager:8000' + + - job_name: 'kg-store' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-store:8000' + + - job_name: 'librarian' + scrape_interval: 5s + static_configs: + - targets: + - 'librarian:8000' + + - job_name: 'mcp-server' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-server:8000' + + - job_name: 'mcp-tool' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-tool:8000' + + - job_name: 'metering' + scrape_interval: 5s + static_configs: + - targets: + - 'metering:8000' + + - job_name: 'metering-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'metering-rag:8000' + + - job_name: 'nlp-query' + scrape_interval: 5s + static_configs: + - targets: + - 'nlp-query:8000' + + - job_name: 'pdf-decoder' + scrape_interval: 5s + static_configs: + - targets: + - 'pdf-decoder:8000' + + - job_name: 'prompt' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt:8000' + + - job_name: 'prompt-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt-rag:8000' + + - job_name: 'query-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-doc-embeddings:8000' + + - job_name: 'query-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-graph-embeddings:8000' + + - job_name: 'query-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'query-objects:8000' + + - job_name: 'query-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'query-triples:8000' + + - job_name: 'store-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-doc-embeddings:8000' + + - job_name: 'store-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-graph-embeddings:8000' + + - job_name: 'store-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'store-objects:8000' + + - job_name: 'store-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'store-triples:8000' + + - job_name: 'structured-query' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'structured-diag' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'text-completion' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion:8000' + + - job_name: 'text-completion-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion-rag:8000' + + # Non-Trustgraph services + - job_name: 'pulsar' + scrape_interval: 5s + static_configs: + - targets: + - 'pulsar:8080' + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/dashboards/log-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/dashboards/log-dashboard.json new file mode 100644 index 00000000..2d40b195 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/dashboards/log-dashboard.json @@ -0,0 +1,178 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 33, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "code", + "expr": "topk(5, sum by (processor) (count_over_time({severity=~\"error|critical\"} [$__auto])))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error volume", + "type": "barchart" + }, + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "builder", + "expr": "{severity=~\"error|critical\"} |= ``", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log errors", + "type": "logs" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Logs", + "uid": "4bdce405-0dd3-4a14-9a8f-792152ebebba", + "version": 13 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/dashboards/overview-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/dashboards/overview-dashboard.json new file mode 100644 index 00000000..31e57fb3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/dashboards/overview-dashboard.json @@ -0,0 +1,1403 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "producer_count_total{processor=\"chunker\",name=\"output\"}\n- ignoring(instance, job, processor, name, status)\nprocessing_count_total{processor=\"kg-extract-relationships\", name=\"input\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Knowledge extraction backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"} >= 2)\nor\n(0 * (producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Graph embeddings backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n) >= 2) or (0 * clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Triples backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(text_completion_duration_bucket[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "99%", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "LLM latency", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 2, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "processing status", + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(status) (rate(processing_count_total{status!=\"success\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Error rate", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(processor) (rate(request_latency_count{processor!=\"config-svc\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "pulsar_msg_backlog{topic!=\"persistent://tg/config/config\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Pub/sub backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(le) (chunk_size_bucket)", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": true, + "legendFormat": "{{le}}", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Chunk size", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate limit events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(process_cpu_seconds_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GB", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "process_resident_memory_bytes / 1073741824", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "hideFrom": { + "viz": false + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by (model) (tokens_total)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Models in use", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(model, direction) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "$", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(direction, model) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Token cost", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "b5c8abf8-fe79-496b-b028-10bde917d1f0", + "version": 1 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/provisioning/dashboard.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/provisioning/dashboard.yml new file mode 100644 index 00000000..9b9e7450 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/provisioning/dashboard.yml @@ -0,0 +1,17 @@ + +apiVersion: 1 + +providers: + + - name: 'trustgraph.ai' + orgId: 1 + folder: 'TrustGraph' + folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/provisioning/datasource.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/provisioning/datasource.yml new file mode 100644 index 00000000..1cf4bc3d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/grafana/provisioning/datasource.yml @@ -0,0 +1,36 @@ +apiVersion: 1 + +prune: true + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129' + + url: http://prometheus:9090 + + basicAuth: false + withCredentials: false + isDefault: true + editable: true + + - name: Loki + type: loki + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'cf6m73l5rnvuod' + + url: http://loki:3100 + + basicAuth: false + withCredentials: false + editable: true + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/loki/local-config.yaml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/loki/local-config.yaml new file mode 100644 index 00000000..8c519ee9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/loki/local-config.yaml @@ -0,0 +1,63 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: debug + grpc_server_max_concurrent_streams: 1000 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + metric_aggregation_enabled: true + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 + +ruler: + alertmanager_url: http://localhost:9093 + +frontend: + encoding: protobuf + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +analytics: + reporting_enabled: false diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/prometheus/prometheus.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/prometheus/prometheus.yml new file mode 100644 index 00000000..b5c8509d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.7/prometheus/prometheus.yml @@ -0,0 +1,247 @@ +global: + + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'trustgraph' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + + # The job name is added as a label `job=` to any timeseries + # scraped from this config. + + # TrustGraph services + + - job_name: 'agent-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'agent-manager:8000' + + - job_name: 'api-gateway' + scrape_interval: 5s + static_configs: + - targets: + - 'api-gateway:8000' + + - job_name: 'chunker' + scrape_interval: 5s + static_configs: + - targets: + - 'chunker:8000' + + - job_name: 'config-svc' + scrape_interval: 5s + static_configs: + - targets: + - 'config-svc:8000' + + - job_name: 'document-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'document-embeddings:8000' + + - job_name: 'document-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'document-rag:8000' + + - job_name: 'embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'embeddings:8000' + + - job_name: 'graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-embeddings:8000' + + - job_name: 'graph-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-rag:8000' + + - job_name: 'kg-extract-agent' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-agent:8000' + + - job_name: 'kg-extract-definitions' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-definitions:8000' + + - job_name: 'kg-extract-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-objects:8000' + + - job_name: 'kg-extract-relationships' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-relationships:8000' + + - job_name: 'kg-extract-ontology' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-ontology:8000' + + - job_name: 'kg-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-manager:8000' + + - job_name: 'kg-store' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-store:8000' + + - job_name: 'librarian' + scrape_interval: 5s + static_configs: + - targets: + - 'librarian:8000' + + - job_name: 'mcp-server' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-server:8000' + + - job_name: 'mcp-tool' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-tool:8000' + + - job_name: 'metering' + scrape_interval: 5s + static_configs: + - targets: + - 'metering:8000' + + - job_name: 'metering-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'metering-rag:8000' + + - job_name: 'nlp-query' + scrape_interval: 5s + static_configs: + - targets: + - 'nlp-query:8000' + + - job_name: 'pdf-decoder' + scrape_interval: 5s + static_configs: + - targets: + - 'pdf-decoder:8000' + + - job_name: 'prompt' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt:8000' + + - job_name: 'prompt-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt-rag:8000' + + - job_name: 'query-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-doc-embeddings:8000' + + - job_name: 'query-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-graph-embeddings:8000' + + - job_name: 'query-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'query-objects:8000' + + - job_name: 'query-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'query-triples:8000' + + - job_name: 'store-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-doc-embeddings:8000' + + - job_name: 'store-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-graph-embeddings:8000' + + - job_name: 'store-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'store-objects:8000' + + - job_name: 'store-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'store-triples:8000' + + - job_name: 'structured-query' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'structured-diag' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'text-completion' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion:8000' + + - job_name: 'text-completion-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion-rag:8000' + + # Non-Trustgraph services + - job_name: 'pulsar' + scrape_interval: 5s + static_configs: + - targets: + - 'pulsar:8080' + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/dashboards/log-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/dashboards/log-dashboard.json new file mode 100644 index 00000000..2d40b195 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/dashboards/log-dashboard.json @@ -0,0 +1,178 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 33, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "code", + "expr": "topk(5, sum by (processor) (count_over_time({severity=~\"error|critical\"} [$__auto])))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error volume", + "type": "barchart" + }, + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "builder", + "expr": "{severity=~\"error|critical\"} |= ``", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log errors", + "type": "logs" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Logs", + "uid": "4bdce405-0dd3-4a14-9a8f-792152ebebba", + "version": 13 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/dashboards/overview-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/dashboards/overview-dashboard.json new file mode 100644 index 00000000..31e57fb3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/dashboards/overview-dashboard.json @@ -0,0 +1,1403 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "producer_count_total{processor=\"chunker\",name=\"output\"}\n- ignoring(instance, job, processor, name, status)\nprocessing_count_total{processor=\"kg-extract-relationships\", name=\"input\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Knowledge extraction backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"} >= 2)\nor\n(0 * (producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Graph embeddings backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n) >= 2) or (0 * clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Triples backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(text_completion_duration_bucket[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "99%", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "LLM latency", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 2, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "processing status", + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(status) (rate(processing_count_total{status!=\"success\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Error rate", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(processor) (rate(request_latency_count{processor!=\"config-svc\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "pulsar_msg_backlog{topic!=\"persistent://tg/config/config\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Pub/sub backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(le) (chunk_size_bucket)", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": true, + "legendFormat": "{{le}}", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Chunk size", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate limit events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(process_cpu_seconds_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GB", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "process_resident_memory_bytes / 1073741824", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "hideFrom": { + "viz": false + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by (model) (tokens_total)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Models in use", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(model, direction) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "$", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(direction, model) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Token cost", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "b5c8abf8-fe79-496b-b028-10bde917d1f0", + "version": 1 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/provisioning/dashboard.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/provisioning/dashboard.yml new file mode 100644 index 00000000..9b9e7450 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/provisioning/dashboard.yml @@ -0,0 +1,17 @@ + +apiVersion: 1 + +providers: + + - name: 'trustgraph.ai' + orgId: 1 + folder: 'TrustGraph' + folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/provisioning/datasource.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/provisioning/datasource.yml new file mode 100644 index 00000000..1cf4bc3d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/grafana/provisioning/datasource.yml @@ -0,0 +1,36 @@ +apiVersion: 1 + +prune: true + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129' + + url: http://prometheus:9090 + + basicAuth: false + withCredentials: false + isDefault: true + editable: true + + - name: Loki + type: loki + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'cf6m73l5rnvuod' + + url: http://loki:3100 + + basicAuth: false + withCredentials: false + editable: true + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/loki/local-config.yaml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/loki/local-config.yaml new file mode 100644 index 00000000..8c519ee9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/loki/local-config.yaml @@ -0,0 +1,63 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: debug + grpc_server_max_concurrent_streams: 1000 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + metric_aggregation_enabled: true + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 + +ruler: + alertmanager_url: http://localhost:9093 + +frontend: + encoding: protobuf + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +analytics: + reporting_enabled: false diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/prometheus/prometheus.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/prometheus/prometheus.yml new file mode 100644 index 00000000..b5c8509d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.8/prometheus/prometheus.yml @@ -0,0 +1,247 @@ +global: + + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'trustgraph' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + + # The job name is added as a label `job=` to any timeseries + # scraped from this config. + + # TrustGraph services + + - job_name: 'agent-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'agent-manager:8000' + + - job_name: 'api-gateway' + scrape_interval: 5s + static_configs: + - targets: + - 'api-gateway:8000' + + - job_name: 'chunker' + scrape_interval: 5s + static_configs: + - targets: + - 'chunker:8000' + + - job_name: 'config-svc' + scrape_interval: 5s + static_configs: + - targets: + - 'config-svc:8000' + + - job_name: 'document-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'document-embeddings:8000' + + - job_name: 'document-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'document-rag:8000' + + - job_name: 'embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'embeddings:8000' + + - job_name: 'graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-embeddings:8000' + + - job_name: 'graph-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-rag:8000' + + - job_name: 'kg-extract-agent' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-agent:8000' + + - job_name: 'kg-extract-definitions' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-definitions:8000' + + - job_name: 'kg-extract-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-objects:8000' + + - job_name: 'kg-extract-relationships' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-relationships:8000' + + - job_name: 'kg-extract-ontology' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-ontology:8000' + + - job_name: 'kg-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-manager:8000' + + - job_name: 'kg-store' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-store:8000' + + - job_name: 'librarian' + scrape_interval: 5s + static_configs: + - targets: + - 'librarian:8000' + + - job_name: 'mcp-server' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-server:8000' + + - job_name: 'mcp-tool' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-tool:8000' + + - job_name: 'metering' + scrape_interval: 5s + static_configs: + - targets: + - 'metering:8000' + + - job_name: 'metering-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'metering-rag:8000' + + - job_name: 'nlp-query' + scrape_interval: 5s + static_configs: + - targets: + - 'nlp-query:8000' + + - job_name: 'pdf-decoder' + scrape_interval: 5s + static_configs: + - targets: + - 'pdf-decoder:8000' + + - job_name: 'prompt' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt:8000' + + - job_name: 'prompt-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt-rag:8000' + + - job_name: 'query-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-doc-embeddings:8000' + + - job_name: 'query-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-graph-embeddings:8000' + + - job_name: 'query-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'query-objects:8000' + + - job_name: 'query-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'query-triples:8000' + + - job_name: 'store-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-doc-embeddings:8000' + + - job_name: 'store-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-graph-embeddings:8000' + + - job_name: 'store-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'store-objects:8000' + + - job_name: 'store-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'store-triples:8000' + + - job_name: 'structured-query' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'structured-diag' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'text-completion' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion:8000' + + - job_name: 'text-completion-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion-rag:8000' + + # Non-Trustgraph services + - job_name: 'pulsar' + scrape_interval: 5s + static_configs: + - targets: + - 'pulsar:8080' + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/dashboards/log-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/dashboards/log-dashboard.json new file mode 100644 index 00000000..2d40b195 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/dashboards/log-dashboard.json @@ -0,0 +1,178 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 33, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "code", + "expr": "topk(5, sum by (processor) (count_over_time({severity=~\"error|critical\"} [$__auto])))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error volume", + "type": "barchart" + }, + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "builder", + "expr": "{severity=~\"error|critical\"} |= ``", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log errors", + "type": "logs" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Logs", + "uid": "4bdce405-0dd3-4a14-9a8f-792152ebebba", + "version": 13 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/dashboards/overview-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/dashboards/overview-dashboard.json new file mode 100644 index 00000000..31e57fb3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/dashboards/overview-dashboard.json @@ -0,0 +1,1403 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "producer_count_total{processor=\"chunker\",name=\"output\"}\n- ignoring(instance, job, processor, name, status)\nprocessing_count_total{processor=\"kg-extract-relationships\", name=\"input\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Knowledge extraction backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"} >= 2)\nor\n(0 * (producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Graph embeddings backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n) >= 2) or (0 * clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Triples backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(text_completion_duration_bucket[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "99%", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "LLM latency", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 2, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "processing status", + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(status) (rate(processing_count_total{status!=\"success\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Error rate", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(processor) (rate(request_latency_count{processor!=\"config-svc\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "pulsar_msg_backlog{topic!=\"persistent://tg/config/config\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Pub/sub backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(le) (chunk_size_bucket)", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": true, + "legendFormat": "{{le}}", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Chunk size", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate limit events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(process_cpu_seconds_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GB", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "process_resident_memory_bytes / 1073741824", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "hideFrom": { + "viz": false + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by (model) (tokens_total)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Models in use", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(model, direction) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "$", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(direction, model) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Token cost", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "b5c8abf8-fe79-496b-b028-10bde917d1f0", + "version": 1 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/provisioning/dashboard.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/provisioning/dashboard.yml new file mode 100644 index 00000000..9b9e7450 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/provisioning/dashboard.yml @@ -0,0 +1,17 @@ + +apiVersion: 1 + +providers: + + - name: 'trustgraph.ai' + orgId: 1 + folder: 'TrustGraph' + folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/provisioning/datasource.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/provisioning/datasource.yml new file mode 100644 index 00000000..1cf4bc3d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/grafana/provisioning/datasource.yml @@ -0,0 +1,36 @@ +apiVersion: 1 + +prune: true + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129' + + url: http://prometheus:9090 + + basicAuth: false + withCredentials: false + isDefault: true + editable: true + + - name: Loki + type: loki + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'cf6m73l5rnvuod' + + url: http://loki:3100 + + basicAuth: false + withCredentials: false + editable: true + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/loki/local-config.yaml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/loki/local-config.yaml new file mode 100644 index 00000000..8c519ee9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/loki/local-config.yaml @@ -0,0 +1,63 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: debug + grpc_server_max_concurrent_streams: 1000 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + metric_aggregation_enabled: true + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 + +ruler: + alertmanager_url: http://localhost:9093 + +frontend: + encoding: protobuf + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +analytics: + reporting_enabled: false diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/prometheus/prometheus.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/prometheus/prometheus.yml new file mode 100644 index 00000000..b5c8509d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/1.9/prometheus/prometheus.yml @@ -0,0 +1,247 @@ +global: + + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'trustgraph' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + + # The job name is added as a label `job=` to any timeseries + # scraped from this config. + + # TrustGraph services + + - job_name: 'agent-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'agent-manager:8000' + + - job_name: 'api-gateway' + scrape_interval: 5s + static_configs: + - targets: + - 'api-gateway:8000' + + - job_name: 'chunker' + scrape_interval: 5s + static_configs: + - targets: + - 'chunker:8000' + + - job_name: 'config-svc' + scrape_interval: 5s + static_configs: + - targets: + - 'config-svc:8000' + + - job_name: 'document-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'document-embeddings:8000' + + - job_name: 'document-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'document-rag:8000' + + - job_name: 'embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'embeddings:8000' + + - job_name: 'graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-embeddings:8000' + + - job_name: 'graph-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-rag:8000' + + - job_name: 'kg-extract-agent' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-agent:8000' + + - job_name: 'kg-extract-definitions' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-definitions:8000' + + - job_name: 'kg-extract-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-objects:8000' + + - job_name: 'kg-extract-relationships' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-relationships:8000' + + - job_name: 'kg-extract-ontology' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-ontology:8000' + + - job_name: 'kg-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-manager:8000' + + - job_name: 'kg-store' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-store:8000' + + - job_name: 'librarian' + scrape_interval: 5s + static_configs: + - targets: + - 'librarian:8000' + + - job_name: 'mcp-server' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-server:8000' + + - job_name: 'mcp-tool' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-tool:8000' + + - job_name: 'metering' + scrape_interval: 5s + static_configs: + - targets: + - 'metering:8000' + + - job_name: 'metering-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'metering-rag:8000' + + - job_name: 'nlp-query' + scrape_interval: 5s + static_configs: + - targets: + - 'nlp-query:8000' + + - job_name: 'pdf-decoder' + scrape_interval: 5s + static_configs: + - targets: + - 'pdf-decoder:8000' + + - job_name: 'prompt' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt:8000' + + - job_name: 'prompt-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt-rag:8000' + + - job_name: 'query-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-doc-embeddings:8000' + + - job_name: 'query-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-graph-embeddings:8000' + + - job_name: 'query-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'query-objects:8000' + + - job_name: 'query-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'query-triples:8000' + + - job_name: 'store-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-doc-embeddings:8000' + + - job_name: 'store-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-graph-embeddings:8000' + + - job_name: 'store-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'store-objects:8000' + + - job_name: 'store-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'store-triples:8000' + + - job_name: 'structured-query' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'structured-diag' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'text-completion' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion:8000' + + - job_name: 'text-completion-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion-rag:8000' + + # Non-Trustgraph services + - job_name: 'pulsar' + scrape_interval: 5s + static_configs: + - targets: + - 'pulsar:8080' + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/dashboards/log-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/dashboards/log-dashboard.json new file mode 100644 index 00000000..2d40b195 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/dashboards/log-dashboard.json @@ -0,0 +1,178 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 33, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "code", + "expr": "topk(5, sum by (processor) (count_over_time({severity=~\"error|critical\"} [$__auto])))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error volume", + "type": "barchart" + }, + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "builder", + "expr": "{severity=~\"error|critical\"} |= ``", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log errors", + "type": "logs" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Logs", + "uid": "4bdce405-0dd3-4a14-9a8f-792152ebebba", + "version": 13 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/dashboards/overview-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/dashboards/overview-dashboard.json new file mode 100644 index 00000000..31e57fb3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/dashboards/overview-dashboard.json @@ -0,0 +1,1403 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "producer_count_total{processor=\"chunker\",name=\"output\"}\n- ignoring(instance, job, processor, name, status)\nprocessing_count_total{processor=\"kg-extract-relationships\", name=\"input\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Knowledge extraction backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"} >= 2)\nor\n(0 * (producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Graph embeddings backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n) >= 2) or (0 * clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Triples backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(text_completion_duration_bucket[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "99%", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "LLM latency", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 2, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "processing status", + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(status) (rate(processing_count_total{status!=\"success\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Error rate", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(processor) (rate(request_latency_count{processor!=\"config-svc\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "pulsar_msg_backlog{topic!=\"persistent://tg/config/config\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Pub/sub backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(le) (chunk_size_bucket)", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": true, + "legendFormat": "{{le}}", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Chunk size", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate limit events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(process_cpu_seconds_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GB", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "process_resident_memory_bytes / 1073741824", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "hideFrom": { + "viz": false + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by (model) (tokens_total)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Models in use", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(model, direction) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "$", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(direction, model) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Token cost", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "b5c8abf8-fe79-496b-b028-10bde917d1f0", + "version": 1 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/provisioning/dashboard.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/provisioning/dashboard.yml new file mode 100644 index 00000000..9b9e7450 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/provisioning/dashboard.yml @@ -0,0 +1,17 @@ + +apiVersion: 1 + +providers: + + - name: 'trustgraph.ai' + orgId: 1 + folder: 'TrustGraph' + folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/provisioning/datasource.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/provisioning/datasource.yml new file mode 100644 index 00000000..1cf4bc3d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/grafana/provisioning/datasource.yml @@ -0,0 +1,36 @@ +apiVersion: 1 + +prune: true + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129' + + url: http://prometheus:9090 + + basicAuth: false + withCredentials: false + isDefault: true + editable: true + + - name: Loki + type: loki + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'cf6m73l5rnvuod' + + url: http://loki:3100 + + basicAuth: false + withCredentials: false + editable: true + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/loki/local-config.yaml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/loki/local-config.yaml new file mode 100644 index 00000000..8c519ee9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/loki/local-config.yaml @@ -0,0 +1,63 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: debug + grpc_server_max_concurrent_streams: 1000 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + metric_aggregation_enabled: true + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 + +ruler: + alertmanager_url: http://localhost:9093 + +frontend: + encoding: protobuf + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +analytics: + reporting_enabled: false diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/prometheus/prometheus.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/prometheus/prometheus.yml new file mode 100644 index 00000000..314fc618 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.0/prometheus/prometheus.yml @@ -0,0 +1,265 @@ +global: + + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'trustgraph' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + + # The job name is added as a label `job=` to any timeseries + # scraped from this config. + + # TrustGraph services + + - job_name: 'agent-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'agent-manager:8000' + + - job_name: 'api-gateway' + scrape_interval: 5s + static_configs: + - targets: + - 'api-gateway:8000' + + - job_name: 'chunker' + scrape_interval: 5s + static_configs: + - targets: + - 'chunker:8000' + + - job_name: 'config-svc' + scrape_interval: 5s + static_configs: + - targets: + - 'config-svc:8000' + + - job_name: 'document-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'document-embeddings:8000' + + - job_name: 'document-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'document-rag:8000' + + - job_name: 'embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'embeddings:8000' + + - job_name: 'graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-embeddings:8000' + + - job_name: 'graph-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-rag:8000' + + - job_name: 'kg-extract-agent' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-agent:8000' + + - job_name: 'kg-extract-definitions' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-definitions:8000' + + - job_name: 'kg-extract-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-objects:8000' + + - job_name: 'kg-extract-relationships' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-relationships:8000' + + - job_name: 'kg-extract-ontology' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-ontology:8000' + + - job_name: 'kg-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-manager:8000' + + - job_name: 'kg-store' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-store:8000' + + - job_name: 'librarian' + scrape_interval: 5s + static_configs: + - targets: + - 'librarian:8000' + + - job_name: 'mcp-server' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-server:8000' + + - job_name: 'mcp-tool' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-tool:8000' + + - job_name: 'metering' + scrape_interval: 5s + static_configs: + - targets: + - 'metering:8000' + + - job_name: 'metering-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'metering-rag:8000' + + - job_name: 'nlp-query' + scrape_interval: 5s + static_configs: + - targets: + - 'nlp-query:8000' + + - job_name: 'pdf-decoder' + scrape_interval: 5s + static_configs: + - targets: + - 'pdf-decoder:8000' + + - job_name: 'prompt' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt:8000' + + - job_name: 'prompt-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt-rag:8000' + + - job_name: 'query-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-doc-embeddings:8000' + + - job_name: 'query-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-graph-embeddings:8000' + + - job_name: 'query-row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-row-embeddings:8000' + + - job_name: 'query-rows' + scrape_interval: 5s + static_configs: + - targets: + - 'query-rows:8000' + + - job_name: 'query-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'query-triples:8000' + + - job_name: 'row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'row-embeddings:8000' + + - job_name: 'store-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-doc-embeddings:8000' + + - job_name: 'store-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-graph-embeddings:8000' + + - job_name: 'store-row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-row-embeddings:8000' + + - job_name: 'store-rows' + scrape_interval: 5s + static_configs: + - targets: + - 'store-rows:8000' + + - job_name: 'store-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'store-triples:8000' + + - job_name: 'structured-query' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'structured-diag' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-diag:8000' + + - job_name: 'text-completion' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion:8000' + + - job_name: 'text-completion-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion-rag:8000' + + # Non-Trustgraph services + - job_name: 'pulsar' + scrape_interval: 5s + static_configs: + - targets: + - 'pulsar:8080' + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/dashboards/log-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/dashboards/log-dashboard.json new file mode 100644 index 00000000..2d40b195 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/dashboards/log-dashboard.json @@ -0,0 +1,178 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 33, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "code", + "expr": "topk(5, sum by (processor) (count_over_time({severity=~\"error|critical\"} [$__auto])))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error volume", + "type": "barchart" + }, + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "builder", + "expr": "{severity=~\"error|critical\"} |= ``", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log errors", + "type": "logs" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Logs", + "uid": "4bdce405-0dd3-4a14-9a8f-792152ebebba", + "version": 13 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/dashboards/overview-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/dashboards/overview-dashboard.json new file mode 100644 index 00000000..31e57fb3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/dashboards/overview-dashboard.json @@ -0,0 +1,1403 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "producer_count_total{processor=\"chunker\",name=\"output\"}\n- ignoring(instance, job, processor, name, status)\nprocessing_count_total{processor=\"kg-extract-relationships\", name=\"input\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Knowledge extraction backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"} >= 2)\nor\n(0 * (producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Graph embeddings backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n) >= 2) or (0 * clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Triples backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(text_completion_duration_bucket[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "99%", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "LLM latency", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 2, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "processing status", + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(status) (rate(processing_count_total{status!=\"success\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Error rate", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(processor) (rate(request_latency_count{processor!=\"config-svc\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "pulsar_msg_backlog{topic!=\"persistent://tg/config/config\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Pub/sub backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(le) (chunk_size_bucket)", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": true, + "legendFormat": "{{le}}", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Chunk size", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate limit events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(process_cpu_seconds_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GB", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "process_resident_memory_bytes / 1073741824", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "hideFrom": { + "viz": false + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by (model) (tokens_total)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Models in use", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(model, direction) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "$", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(direction, model) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Token cost", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview", + "uid": "b5c8abf8-fe79-496b-b028-10bde917d1f0", + "version": 1 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/provisioning/dashboard.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/provisioning/dashboard.yml new file mode 100644 index 00000000..9b9e7450 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/provisioning/dashboard.yml @@ -0,0 +1,17 @@ + +apiVersion: 1 + +providers: + + - name: 'trustgraph.ai' + orgId: 1 + folder: 'TrustGraph' + folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/provisioning/datasource.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/provisioning/datasource.yml new file mode 100644 index 00000000..1cf4bc3d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/grafana/provisioning/datasource.yml @@ -0,0 +1,36 @@ +apiVersion: 1 + +prune: true + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129' + + url: http://prometheus:9090 + + basicAuth: false + withCredentials: false + isDefault: true + editable: true + + - name: Loki + type: loki + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'cf6m73l5rnvuod' + + url: http://loki:3100 + + basicAuth: false + withCredentials: false + editable: true + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/loki/local-config.yaml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/loki/local-config.yaml new file mode 100644 index 00000000..8c519ee9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/loki/local-config.yaml @@ -0,0 +1,63 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: debug + grpc_server_max_concurrent_streams: 1000 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + metric_aggregation_enabled: true + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 + +ruler: + alertmanager_url: http://localhost:9093 + +frontend: + encoding: protobuf + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +analytics: + reporting_enabled: false diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/prometheus/prometheus.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/prometheus/prometheus.yml new file mode 100644 index 00000000..314fc618 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.1/prometheus/prometheus.yml @@ -0,0 +1,265 @@ +global: + + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'trustgraph' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + + # The job name is added as a label `job=` to any timeseries + # scraped from this config. + + # TrustGraph services + + - job_name: 'agent-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'agent-manager:8000' + + - job_name: 'api-gateway' + scrape_interval: 5s + static_configs: + - targets: + - 'api-gateway:8000' + + - job_name: 'chunker' + scrape_interval: 5s + static_configs: + - targets: + - 'chunker:8000' + + - job_name: 'config-svc' + scrape_interval: 5s + static_configs: + - targets: + - 'config-svc:8000' + + - job_name: 'document-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'document-embeddings:8000' + + - job_name: 'document-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'document-rag:8000' + + - job_name: 'embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'embeddings:8000' + + - job_name: 'graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-embeddings:8000' + + - job_name: 'graph-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-rag:8000' + + - job_name: 'kg-extract-agent' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-agent:8000' + + - job_name: 'kg-extract-definitions' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-definitions:8000' + + - job_name: 'kg-extract-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-objects:8000' + + - job_name: 'kg-extract-relationships' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-relationships:8000' + + - job_name: 'kg-extract-ontology' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-ontology:8000' + + - job_name: 'kg-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-manager:8000' + + - job_name: 'kg-store' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-store:8000' + + - job_name: 'librarian' + scrape_interval: 5s + static_configs: + - targets: + - 'librarian:8000' + + - job_name: 'mcp-server' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-server:8000' + + - job_name: 'mcp-tool' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-tool:8000' + + - job_name: 'metering' + scrape_interval: 5s + static_configs: + - targets: + - 'metering:8000' + + - job_name: 'metering-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'metering-rag:8000' + + - job_name: 'nlp-query' + scrape_interval: 5s + static_configs: + - targets: + - 'nlp-query:8000' + + - job_name: 'pdf-decoder' + scrape_interval: 5s + static_configs: + - targets: + - 'pdf-decoder:8000' + + - job_name: 'prompt' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt:8000' + + - job_name: 'prompt-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt-rag:8000' + + - job_name: 'query-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-doc-embeddings:8000' + + - job_name: 'query-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-graph-embeddings:8000' + + - job_name: 'query-row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-row-embeddings:8000' + + - job_name: 'query-rows' + scrape_interval: 5s + static_configs: + - targets: + - 'query-rows:8000' + + - job_name: 'query-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'query-triples:8000' + + - job_name: 'row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'row-embeddings:8000' + + - job_name: 'store-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-doc-embeddings:8000' + + - job_name: 'store-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-graph-embeddings:8000' + + - job_name: 'store-row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-row-embeddings:8000' + + - job_name: 'store-rows' + scrape_interval: 5s + static_configs: + - targets: + - 'store-rows:8000' + + - job_name: 'store-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'store-triples:8000' + + - job_name: 'structured-query' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'structured-diag' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-diag:8000' + + - job_name: 'text-completion' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion:8000' + + - job_name: 'text-completion-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion-rag:8000' + + # Non-Trustgraph services + - job_name: 'pulsar' + scrape_interval: 5s + static_configs: + - targets: + - 'pulsar:8080' + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/dashboards/log-dashboard.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/dashboards/log-dashboard.json new file mode 100644 index 00000000..2d40b195 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/dashboards/log-dashboard.json @@ -0,0 +1,178 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 33, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "never", + "stacking": "normal", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "code", + "expr": "topk(5, sum by (processor) (count_over_time({severity=~\"error|critical\"} [$__auto])))", + "queryType": "range", + "refId": "A" + } + ], + "title": "Error volume", + "type": "barchart" + }, + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cf6m73l5rnvuod" + }, + "direction": "backward", + "editorMode": "builder", + "expr": "{severity=~\"error|critical\"} |= ``", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log errors", + "type": "logs" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Logs", + "uid": "4bdce405-0dd3-4a14-9a8f-792152ebebba", + "version": 13 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/dashboards/overview-dashboard-pulsar.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/dashboards/overview-dashboard-pulsar.json new file mode 100644 index 00000000..d4f0f1af --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/dashboards/overview-dashboard-pulsar.json @@ -0,0 +1,1403 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "producer_count_total{processor=\"chunker\",name=\"output\"}\n- ignoring(instance, job, processor, name, status)\nprocessing_count_total{processor=\"kg-extract-relationships\", name=\"input\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Knowledge extraction backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"} >= 2)\nor\n(0 * (producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Graph embeddings backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n) >= 2) or (0 * clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Triples backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(text_completion_duration_bucket[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "99%", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "LLM latency", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 2, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "processing status", + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(status) (rate(processing_count_total{status!=\"success\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Error rate", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(processor) (rate(request_latency_count{processor!=\"config-svc\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "pulsar_msg_backlog{topic!=\"persistent://tg/config/config\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Pub/sub backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(le) (chunk_size_bucket)", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": true, + "legendFormat": "{{le}}", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Chunk size", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate limit events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(process_cpu_seconds_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GB", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "process_resident_memory_bytes / 1073741824", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "hideFrom": { + "viz": false + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by (model) (tokens_total)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Models in use", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(model, direction) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "$", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(direction, model) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Token cost", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview (Pulsar)", + "uid": "ad77jv9", + "version": 1 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/dashboards/overview-dashboard-rabbitmq.json b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/dashboards/overview-dashboard-rabbitmq.json new file mode 100644 index 00000000..c3dee6d8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/dashboards/overview-dashboard-rabbitmq.json @@ -0,0 +1,1403 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "producer_count_total{processor=\"chunker\",name=\"output\"}\n- ignoring(instance, job, processor, name, status)\nprocessing_count_total{processor=\"kg-extract-relationships\", name=\"input\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Knowledge extraction backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"} >= 2)\nor\n(0 * (producer_count_total{processor=\"graph-embeddings\",name=\"output\"} - ignoring(instance, job, processor, name, status) processing_count_total{processor=\"ge-write\",name=\"input\"}))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Graph embeddings backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "expr": "(clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n) >= 2) or (0 * clamp_min(\n (\n producer_count_total{processor=\"kg-extract-definitions\",name=\"triples\"}\n + on(flow)\n producer_count_total{processor=\"kg-extract-relationships\",name=\"triples\"}\n )\n - on(flow) processing_count_total{processor=\"triples-write\",name=\"input\"},\n 0\n))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Triples backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(le) (rate(text_completion_duration_bucket[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "99%", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "LLM latency", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 2, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisLabel": "processing status", + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(status) (rate(processing_count_total{status!=\"success\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Error rate", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(processor) (rate(request_latency_count{processor!=\"config-svc\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": " label_replace( \n rabbitmq_queue_messages{queue!~\"tg.state.config.*|amq.gen-.*\"}, \n \"short\", \"$2:$1:$3\", \"queue\", \"^([^.]+)\\\\.([^.]+)\\\\.([^.]+)\\\\..+$\" \n ) \n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{short}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Pub/sub backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-green", + "mode": "palette-classic-by-name" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "max by(le) (chunk_size_bucket)", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": true, + "legendFormat": "{{le}}", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Chunk size", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(job) (increase(rate_limit_count_total[$__rate_interval]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Rate limit events", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "rate(process_cpu_seconds_total[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GB", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "process_resident_memory_bytes / 1073741824", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "hideFrom": { + "viz": false + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "sum by (model) (tokens_total)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Models in use", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": {}, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 39 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(model, direction) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Tokens", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "$", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 39 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "f6b18033-5918-4e05-a1ca-4cb30343b129" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "sum by(direction, model) (rate(tokens_total[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{model}} - {{direction}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Token cost", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 42, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Overview (RabbitMQ)", + "uid": "adkxj6x", + "version": 1 +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/provisioning/dashboard.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/provisioning/dashboard.yml new file mode 100644 index 00000000..9b9e7450 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/provisioning/dashboard.yml @@ -0,0 +1,17 @@ + +apiVersion: 1 + +providers: + + - name: 'trustgraph.ai' + orgId: 1 + folder: 'TrustGraph' + folderUid: 'b6c5be90-d432-4df8-aeab-737c7b151228' + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/provisioning/datasource.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/provisioning/datasource.yml new file mode 100644 index 00000000..1cf4bc3d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/grafana/provisioning/datasource.yml @@ -0,0 +1,36 @@ +apiVersion: 1 + +prune: true + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'f6b18033-5918-4e05-a1ca-4cb30343b129' + + url: http://prometheus:9090 + + basicAuth: false + withCredentials: false + isDefault: true + editable: true + + - name: Loki + type: loki + access: proxy + orgId: 1 + # Sets a custom UID to reference this + # data source in other parts of the configuration. + # If not specified, Grafana generates one. + uid: 'cf6m73l5rnvuod' + + url: http://loki:3100 + + basicAuth: false + withCredentials: false + editable: true + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/loki/local-config.yaml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/loki/local-config.yaml new file mode 100644 index 00000000..8c519ee9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/loki/local-config.yaml @@ -0,0 +1,63 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: debug + grpc_server_max_concurrent_streams: 1000 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +limits_config: + metric_aggregation_enabled: true + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +pattern_ingester: + enabled: true + metric_aggregation: + loki_address: localhost:3100 + +ruler: + alertmanager_url: http://localhost:9093 + +frontend: + encoding: protobuf + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +analytics: + reporting_enabled: false diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/prometheus/prometheus-pulsar.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/prometheus/prometheus-pulsar.yml new file mode 100644 index 00000000..2413fb41 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/prometheus/prometheus-pulsar.yml @@ -0,0 +1,271 @@ +global: + + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'trustgraph' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + + # The job name is added as a label `job=` to any timeseries + # scraped from this config. + + # TrustGraph services + + - job_name: 'agent-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'agent-manager:8000' + + - job_name: 'api-gateway' + scrape_interval: 5s + static_configs: + - targets: + - 'api-gateway:8000' + + - job_name: 'chunker' + scrape_interval: 5s + static_configs: + - targets: + - 'chunker:8000' + + - job_name: 'config-svc' + scrape_interval: 5s + static_configs: + - targets: + - 'config-svc:8000' + + - job_name: 'document-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'document-embeddings:8000' + + - job_name: 'document-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'document-rag:8000' + + - job_name: 'embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'embeddings:8000' + + - job_name: 'graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-embeddings:8000' + + - job_name: 'graph-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-rag:8000' + + - job_name: 'kg-extract-agent' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-agent:8000' + + - job_name: 'kg-extract-definitions' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-definitions:8000' + + - job_name: 'kg-extract-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-objects:8000' + + - job_name: 'kg-extract-relationships' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-relationships:8000' + + - job_name: 'kg-extract-ontology' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-ontology:8000' + + - job_name: 'kg-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-manager:8000' + + - job_name: 'kg-store' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-store:8000' + + - job_name: 'librarian' + scrape_interval: 5s + static_configs: + - targets: + - 'librarian:8000' + + - job_name: 'mcp-server' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-server:8000' + + - job_name: 'mcp-tool' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-tool:8000' + + - job_name: 'metering' + scrape_interval: 5s + static_configs: + - targets: + - 'metering:8000' + + - job_name: 'metering-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'metering-rag:8000' + + - job_name: 'nlp-query' + scrape_interval: 5s + static_configs: + - targets: + - 'nlp-query:8000' + + - job_name: 'pdf-decoder' + scrape_interval: 5s + static_configs: + - targets: + - 'pdf-decoder:8000' + + - job_name: 'prompt' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt:8000' + + - job_name: 'prompt-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt-rag:8000' + + - job_name: 'query-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-doc-embeddings:8000' + + - job_name: 'query-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-graph-embeddings:8000' + + - job_name: 'query-row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-row-embeddings:8000' + + - job_name: 'query-rows' + scrape_interval: 5s + static_configs: + - targets: + - 'query-rows:8000' + + - job_name: 'query-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'query-triples:8000' + + - job_name: 'row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'row-embeddings:8000' + + - job_name: 'store-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-doc-embeddings:8000' + + - job_name: 'store-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-graph-embeddings:8000' + + - job_name: 'store-row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-row-embeddings:8000' + + - job_name: 'store-rows' + scrape_interval: 5s + static_configs: + - targets: + - 'store-rows:8000' + + - job_name: 'store-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'store-triples:8000' + + - job_name: 'structured-query' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'structured-diag' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-diag:8000' + + - job_name: 'text-completion' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion:8000' + + - job_name: 'text-completion-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion-rag:8000' + + # Non-Trustgraph services + - job_name: 'garage' + metrics_path: '/metrics' + scrape_interval: 5s + static_configs: + - targets: + - 'garage:3903' + + - job_name: 'pulsar' + scrape_interval: 5s + static_configs: + - targets: + - 'pulsar:8080' diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/prometheus/prometheus-rabbitmq.yml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/prometheus/prometheus-rabbitmq.yml new file mode 100644 index 00000000..08895dd7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/2.2/prometheus/prometheus-rabbitmq.yml @@ -0,0 +1,272 @@ +global: + + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'trustgraph' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + + # The job name is added as a label `job=` to any timeseries + # scraped from this config. + + # TrustGraph services + + - job_name: 'agent-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'agent-manager:8000' + + - job_name: 'api-gateway' + scrape_interval: 5s + static_configs: + - targets: + - 'api-gateway:8000' + + - job_name: 'chunker' + scrape_interval: 5s + static_configs: + - targets: + - 'chunker:8000' + + - job_name: 'config-svc' + scrape_interval: 5s + static_configs: + - targets: + - 'config-svc:8000' + + - job_name: 'document-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'document-embeddings:8000' + + - job_name: 'document-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'document-rag:8000' + + - job_name: 'embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'embeddings:8000' + + - job_name: 'graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-embeddings:8000' + + - job_name: 'graph-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'graph-rag:8000' + + - job_name: 'kg-extract-agent' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-agent:8000' + + - job_name: 'kg-extract-definitions' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-definitions:8000' + + - job_name: 'kg-extract-objects' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-objects:8000' + + - job_name: 'kg-extract-relationships' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-relationships:8000' + + - job_name: 'kg-extract-ontology' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-extract-ontology:8000' + + - job_name: 'kg-manager' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-manager:8000' + + - job_name: 'kg-store' + scrape_interval: 5s + static_configs: + - targets: + - 'kg-store:8000' + + - job_name: 'librarian' + scrape_interval: 5s + static_configs: + - targets: + - 'librarian:8000' + + - job_name: 'mcp-server' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-server:8000' + + - job_name: 'mcp-tool' + scrape_interval: 5s + static_configs: + - targets: + - 'mcp-tool:8000' + + - job_name: 'metering' + scrape_interval: 5s + static_configs: + - targets: + - 'metering:8000' + + - job_name: 'metering-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'metering-rag:8000' + + - job_name: 'nlp-query' + scrape_interval: 5s + static_configs: + - targets: + - 'nlp-query:8000' + + - job_name: 'pdf-decoder' + scrape_interval: 5s + static_configs: + - targets: + - 'pdf-decoder:8000' + + - job_name: 'prompt' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt:8000' + + - job_name: 'prompt-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'prompt-rag:8000' + + - job_name: 'query-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-doc-embeddings:8000' + + - job_name: 'query-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-graph-embeddings:8000' + + - job_name: 'query-row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'query-row-embeddings:8000' + + - job_name: 'query-rows' + scrape_interval: 5s + static_configs: + - targets: + - 'query-rows:8000' + + - job_name: 'query-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'query-triples:8000' + + - job_name: 'row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'row-embeddings:8000' + + - job_name: 'store-doc-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-doc-embeddings:8000' + + - job_name: 'store-graph-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-graph-embeddings:8000' + + - job_name: 'store-row-embeddings' + scrape_interval: 5s + static_configs: + - targets: + - 'store-row-embeddings:8000' + + - job_name: 'store-rows' + scrape_interval: 5s + static_configs: + - targets: + - 'store-rows:8000' + + - job_name: 'store-triples' + scrape_interval: 5s + static_configs: + - targets: + - 'store-triples:8000' + + - job_name: 'structured-query' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-query:8000' + + - job_name: 'structured-diag' + scrape_interval: 5s + static_configs: + - targets: + - 'structured-diag:8000' + + - job_name: 'text-completion' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion:8000' + + - job_name: 'text-completion-rag' + scrape_interval: 5s + static_configs: + - targets: + - 'text-completion-rag:8000' + + # Non-Trustgraph services + - job_name: 'garage' + metrics_path: '/metrics' + scrape_interval: 5s + static_configs: + - targets: + - 'garage:3903' + + - job_name: 'rabbitmq' + metrics_path: '/metrics/per-object' + scrape_interval: 5s + static_configs: + - targets: + - 'rabbitmq:15692' diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/deploy/docker-compose-deploy.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/deploy/docker-compose-deploy.md new file mode 100644 index 00000000..5a474399 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/deploy/docker-compose-deploy.md @@ -0,0 +1,25 @@ +When you download the deploy configuration, you will have a ZIP file containing all the configuration needed to launch TrustGraph in Docker Compose. Unzip the ZIP file: + +```bash +unzip deploy.zip +``` + +On MacOS, it may be necessary to specify a destination directory for the TrustGraph package: + +```bash +unzip deploy.zip -d deploy +``` + +Navigate to the `docker-compose` directory. From this directory, launch TrustGraph with: + +```bash +docker compose -f docker-compose.yaml up -d +``` + +If you are on Linux, running SELinux, you may need to change permissions on files in the deploy bundle so that they are accessible from within containers. This affects the `grafana` and `prometheus` directories. + +```bash +chcon -Rt svirt_sandbox_file_t grafana prometheus +chmod 755 prometheus/ grafana/ grafana/*/ +chmod 644 prometheus/* grafana/*/* +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/deploy/k8s-deploy.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/deploy/k8s-deploy.md new file mode 100644 index 00000000..ea442c1f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/deploy/k8s-deploy.md @@ -0,0 +1,13 @@ +To deploy to Kubernetes, you likely need to have Kubernetes credentials set up to connect to the Kubernetes management service. The mechanism to do this varies with the different kinds of Kubernetes services in use, check with your cloud provider documentation. + +When you download the deploy configuration, you will have a ZIP file containing all the configuration needed to launch TrustGraph on Kubernetes. Unzip the ZIP file: + +```bash +unzip deploy.zip +``` + +and launch: + +```bash +kubectl apply -f resources.yaml +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/deploy/podman-compose-deploy.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/deploy/podman-compose-deploy.md new file mode 100644 index 00000000..213664b8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/deploy/podman-compose-deploy.md @@ -0,0 +1,19 @@ +When you download the deploy configuration, you will have a ZIP file containing all the configuration needed to launch TrustGraph in Podman Compose. Unzip the ZIP file: + +```bash +unzip deploy.zip +``` + +Navigate to the `docker-compose` directory. From this directory, launch TrustGraph with: + +```bash +podman compose -f docker-compose.yaml up -d +``` + +If you are on Linux, running SELinux, you may need to change permissions on files in the deploy bundle so that they are accessible from within containers. This affects the `grafana` and `prometheus` directories. + +```bash +chcon -Rt svirt_sandbox_file_t grafana prometheus +chmod 755 prometheus/ grafana/ grafana/*/ +chmod 644 prometheus/* grafana/*/* +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/features/document-rag.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/features/document-rag.md new file mode 100644 index 00000000..2d7ac07f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/features/document-rag.md @@ -0,0 +1,5 @@ +Document RAG APIs are separate from GraphRAG. You can use `tg-invoke-document-rag` to test Document RAG processing once documents are loaded: + +```bash +tg-invoke-document-rag -q "Describe a cat" +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/features/workbench.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/features/workbench.md new file mode 100644 index 00000000..75f1f8e6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/features/workbench.md @@ -0,0 +1,5 @@ +Once the system is running, you can access the Workbench on port 8888, or access using the following URL: + +[http://localhost:8888/](http://localhost:8888/) + +Once you have data loaded, you can present a Graph RAG query on the Chat tab. As well as answering the question, a list of semantic relationships which were used to answer the question are shown and these can be used to navigate the knowledge graph. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/gateway-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/gateway-compose.md new file mode 100644 index 00000000..c499a823 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/gateway-compose.md @@ -0,0 +1,5 @@ +The API Gateway is a required component which supports the CLI and Workbench. The API Gateway must be configured with a secret key using an environment variable. The secret can be set to an empty string if no authentication is required. + +``` +GATEWAY_SECRET= +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/gateway-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/gateway-k8s.md new file mode 100644 index 00000000..ee3f4186 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/gateway-k8s.md @@ -0,0 +1,7 @@ +The API Gateway is a required component which supports the CLI and Workbench. The API Gateway must be configured with a secret key. However, that secret key can be empty if no authentication is required. The Workbench does not currently use keys for authentication. The below example shows how to set the API Gateway secret to be empty with no authentication. + +```bash +kubectl -n {{namespace}} create secret \ + generic gateway-secret \ + --from-literal=gateway-secret= +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/mcp-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/mcp-compose.md new file mode 100644 index 00000000..61121d52 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/mcp-compose.md @@ -0,0 +1,6 @@ +The MCP server requires two secrets provided as environment variables. The MCP server secret is used by clients to authenticate to the MCP service. The gateway secret must match the value configured for the API Gateway, as the MCP server acts as a client of the gateway. Both can be set to empty strings to disable authentication. + +``` +MCP_SERVER_SECRET= +GATEWAY_SECRET= +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/mcp-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/mcp-k8s.md new file mode 100644 index 00000000..7b0119fd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/gateway/mcp-k8s.md @@ -0,0 +1,8 @@ +The MCP server requires two secrets provided in a Kubernetes secret. The MCP server secret is used by clients to authenticate to the MCP service. The gateway secret must match the value configured for the API Gateway, as the MCP server acts as a client of the gateway. Both can be set to empty strings to disable authentication. + +```bash +kubectl -n {{namespace}} create secret \ + generic mcp-server-secret \ + --from-literal=mcp-server-secret= \ + --from-literal=gateway-secret= +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-compose.md new file mode 100644 index 00000000..a34ae9d7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-compose.md @@ -0,0 +1,6 @@ +To use Azure Serverless APIs, you need to have a serverless endpoint deployed, and you must also provide an endpoint token as an environment variable. + +``` +AZURE_ENDPOINT=https://ENDPOINT.API.HOST.GOES.HERE/ +AZURE_TOKEN=TOKEN-GOES-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-k8s.md new file mode 100644 index 00000000..b213fa20 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-k8s.md @@ -0,0 +1,8 @@ +To use Azure Serverless APIs, you need to have a serverless endpoint deployed. You must also provide an Azure endpoint and token in a Kubernetes secret before launching the application. + +```bash +kubectl -n {{namespace}} create secret \ + generic azure-credentials \ + --from-literal=azure-endpoint=AZURE-ENDPOINT \ + --from-literal=azure-token=AZURE-TOKEN +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-openai-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-openai-compose.md new file mode 100644 index 00000000..9b1fe641 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-openai-compose.md @@ -0,0 +1,8 @@ +To use Azure's OpenAI APIs, you need to have a serverless OpenAI endpoint deployed, and you must also provide an endpoint token as an environment variable. In addition, the OpenAI API requires an API Version and Model Name to be set. The Model Name is set by the user during the deployment within AzureAI. + +``` +AZURE_ENDPOINT=https://ENDPOINT.API.HOST.GOES.HERE/ +AZURE_TOKEN=TOKEN-GOES-HERE +AZURE_API_VERSION=API-VERSION-GOES-HERE +AZURE_MODEL=MODEL-NAME-GOES-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-openai-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-openai-k8s.md new file mode 100644 index 00000000..fce4c097 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/azure-openai-k8s.md @@ -0,0 +1,10 @@ +To use Azure's OpenAI APIs, you need to have a serverless OpenAI endpoint deployed. You must also provide an endpoint token, API version, and model name in a Kubernetes secret. The Model Name is set by the user during the deployment within AzureAI. + +```bash +kubectl -n {{namespace}} create secret \ + generic azure-openai-credentials \ + --from-literal=azure-endpoint=https://ENDPOINT.API.HOST.GOES.HERE/ \ + --from-literal=azure-token=TOKEN-GOES-HERE \ + --from-literal=azure-api-version=API-VERSION-GOES-HERE \ + --from-literal=azure-model=MODEL-NAME-GOES-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/bedrock-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/bedrock-compose.md new file mode 100644 index 00000000..d102d115 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/bedrock-compose.md @@ -0,0 +1,7 @@ +To use AWS Bedrock, you must have enabled models in the AWS Bedrock console. You must also provide an AWS access key ID and secret key. + +``` +AWS_ACCESS_KEY_ID=ID-KEY-HERE +AWS_SECRET_ACCESS_KEY=TOKEN-GOES-HERE +AWS_DEFAULT_REGION=AWS-REGION-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/bedrock-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/bedrock-k8s.md new file mode 100644 index 00000000..13a994e0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/bedrock-k8s.md @@ -0,0 +1,9 @@ +To use AWS Bedrock, you must have enabled models in the AWS Bedrock console. You must also provide an AWS access key ID and secret key as a Kubernetes secret before deploying the application. + +```bash +kubectl -n {{namespace}} create secret \ + generic bedrock-credentials \ + --from-literal=aws-id-key=AWS-ID-KEY \ + --from-literal=aws-secret=AWS-SECRET-KEY \ + --from-literal=aws-region=AWS-REGION-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/claude-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/claude-compose.md new file mode 100644 index 00000000..02327a36 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/claude-compose.md @@ -0,0 +1,5 @@ +To use Anthropic Claude, you need a Claude API key. Provide the Claude API key in an environment variable when running the Docker Compose configuration. + +``` +CLAUDE_KEY=CLAUDE-KEY-GOES-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/claude-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/claude-k8s.md new file mode 100644 index 00000000..b07aec0c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/claude-k8s.md @@ -0,0 +1,7 @@ +To use Anthropic Claude, you need a Claude API key which must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} create secret \ + generic claude-credentials \ + --from-literal=claude-key=CLAUDE-KEY-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/cohere-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/cohere-compose.md new file mode 100644 index 00000000..7a29337a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/cohere-compose.md @@ -0,0 +1,5 @@ +To use Cohere APIs, you need an API token which must be provided in an environment variable. + +``` +COHERE_KEY=TOKEN-GOES-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/cohere-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/cohere-k8s.md new file mode 100644 index 00000000..0be46578 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/cohere-k8s.md @@ -0,0 +1,7 @@ +To use Cohere APIs, you need an API token which must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} create secret \ + generic cohere-credentials \ + --from-literal=cohere-key=COHERE-KEY +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/google-ai-studio-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/google-ai-studio-compose.md new file mode 100644 index 00000000..ec4acb75 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/google-ai-studio-compose.md @@ -0,0 +1,5 @@ +To use Google AI Studio APIs, you need an API token which must be provided in an environment variable. + +``` +GOOGLE_AI_STUDIO_KEY=TOKEN-GOES-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/google-ai-studio-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/google-ai-studio-k8s.md new file mode 100644 index 00000000..5a3e77cf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/google-ai-studio-k8s.md @@ -0,0 +1,7 @@ +To use Google AI Studio APIs, you need an API token which must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} create secret \ + generic googleaistudio-credentials \ + --from-literal=google-ai-studio-key=GOOGLEAISTUDIO-KEY +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/llamafile-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/llamafile-compose.md new file mode 100644 index 00000000..c0ceb3b0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/llamafile-compose.md @@ -0,0 +1,5 @@ +To use Llamafile, you must have a Llamafile service running on an accessible host. The Llamafile host must be provided in an environment variable. + +``` +LLAMAFILE_URL=LLAMAFILE-URL +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/llamafile-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/llamafile-k8s.md new file mode 100644 index 00000000..93e5d88b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/llamafile-k8s.md @@ -0,0 +1,7 @@ +To use Llamafile, you must have a Llamafile service running on an accessible host. The Llamafile host must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} create secret \ + generic llamafile-credentials \ + --from-literal=llamafile-url=http://llamafile:1234/ +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/lm-studio-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/lm-studio-compose.md new file mode 100644 index 00000000..291a46a5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/lm-studio-compose.md @@ -0,0 +1,9 @@ +LMStudio allows you to run models locally, with a nice UX. The LMStudio application or service must be running, and have the REST API enabled, and model made available by pulling from the model repository. + +Note that LMStudio is a commercial product - a licence is needed for non-personal usage. See [lmstudio.ai/work](https://lmstudio.ai/work). + +``` +LMSTUDIO_URL=http://localhost:1234 +``` + +Replace the URL with the URL of your LMStudio API service. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/lm-studio-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/lm-studio-k8s.md new file mode 100644 index 00000000..6e27c233 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/lm-studio-k8s.md @@ -0,0 +1,9 @@ +LMStudio service URL must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} \ + create secret generic lmstudio-credentials \ + --from-literal=lmstudio-url=http://lmstudio:11434/ +``` + +Replace the URL with the URL of your LMStudio service. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/mistral-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/mistral-compose.md new file mode 100644 index 00000000..9dce9c86 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/mistral-compose.md @@ -0,0 +1,5 @@ +To use Mistral, you need a Mistral API key. Provide the Mistral API key in an environment variable when running the Docker Compose configuration. + +``` +MISTRAL_TOKEN=TOKEN-GOES-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/mistral-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/mistral-k8s.md new file mode 100644 index 00000000..c19e7324 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/mistral-k8s.md @@ -0,0 +1,7 @@ +To use Mistral, you need a Mistral API key which must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} create secret \ + generic mistral-credentials \ + --from-literal=mistral-key=MISTRAL-TOKEN +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/ollama-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/ollama-compose.md new file mode 100644 index 00000000..0804b848 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/ollama-compose.md @@ -0,0 +1,9 @@ +The power of Ollama is the flexibility it provides in Language Model deployments. Being able to run LMs with Ollama enables fully secure AI TrustGraph pipelines that aren't relying on any external APIs. No data is leaving the host environment or network. + +The Ollama service must be running, and have required models available using `ollama pull`. The Ollama service URL must be provided in an environment variable. + +``` +OLLAMA_HOST=http://ollama-host:11434 +``` + +Replace the URL with the URL of your Ollama service. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/ollama-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/ollama-k8s.md new file mode 100644 index 00000000..5ccabea4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/ollama-k8s.md @@ -0,0 +1,9 @@ +The Ollama service URL must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} \ + create secret generic ollama-credentials \ + --from-literal=ollama-host=http://ollama:11434/ +``` + +Replace the URL with the URL of your Ollama service. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/openai-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/openai-compose.md new file mode 100644 index 00000000..98136288 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/openai-compose.md @@ -0,0 +1,5 @@ +To use OpenAI APIs, you need an API token which must be provided in an environment variable. + +``` +OPENAI_TOKEN=TOKEN-GOES-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/openai-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/openai-k8s.md new file mode 100644 index 00000000..040d98a6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/openai-k8s.md @@ -0,0 +1,7 @@ +To use OpenAI APIs, you need an API token which must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} create secret \ + generic openai-credentials \ + --from-literal=openai-token=OPENAI-TOKEN-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/tgi-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/tgi-compose.md new file mode 100644 index 00000000..3c9807fe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/tgi-compose.md @@ -0,0 +1,9 @@ +Text Generation Inference (TGI) is Hugging Face's production-ready inference server for LLMs. It provides high-performance text generation with features like continuous batching, tensor parallelism, and optimized attention mechanisms. + +The TGI service must be running with the required model loaded. The TGI service URL must be provided in an environment variable. + +``` +TGI_BASE_URL=http://tgi-host:8080/v1 +``` + +Replace the URL with the URL of your TGI service, noting the `v1` suffix for OpenAI-compatible API. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/tgi-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/tgi-k8s.md new file mode 100644 index 00000000..ac78434a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/tgi-k8s.md @@ -0,0 +1,9 @@ +The TGI service URL must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} \ + create secret generic tgi-credentials \ + --from-literal=tgi-url=http://tgi:8080/v1 +``` + +Replace the URL with the URL of your TGI service. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vertex-ai-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vertex-ai-compose.md new file mode 100644 index 00000000..03fcc98d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vertex-ai-compose.md @@ -0,0 +1,7 @@ +To use VertexAI, you need to have a Google Cloud credential file provisioned for a service account which has access to the VertexAI services. This means signing up to GCP and using an existing, or launching a new GCP project. The GCP credential will be a JSON file which should be stored in `vertexai/private.json`. + +The credential file is mounted as a volume in Docker Compose, which can cause issues with SELinux if you are running on Linux. Make sure that Docker has access to volume files if this affects you. + +```bash +chcon -Rt svirt_sandbox_file_t vertexai/ +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vertex-ai-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vertex-ai-k8s.md new file mode 100644 index 00000000..5d067e36 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vertex-ai-k8s.md @@ -0,0 +1,10 @@ +To use VertexAI, you need to have a Google Cloud credential file provisioned for a service account which has access to the VertexAI services. This means signing up to GCP and using an existing, or launching a new GCP project. The GCP credential will be a JSON file which would arrive in a file called `private.json`. + +The private.json file should be loaded into Kubernetes as a secret. + +```bash +kubectl -n {{namespace}} create secret \ + generic vertexai-creds --from-file=private.json=private.json +``` + +> **Warning:** Google Cloud private.json files are secrets which potentially provide access to all of your Google Cloud resources. Take great care to ensure that the permissions of the account are minimal, ideally scoped to just AI services. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vllm-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vllm-compose.md new file mode 100644 index 00000000..cd41cca1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vllm-compose.md @@ -0,0 +1,9 @@ +vLLM is a high-throughput, memory-efficient inference and serving engine for LLMs. Using PagedAttention and continuous batching, vLLM enables fully secure AI TrustGraph pipelines that aren't relying on any external APIs. No data is leaving the host environment or network. + +The vLLM service must be running with the required model loaded using `vllm serve`. The vLLM service URL must be provided in an environment variable. + +``` +VLLM_BASE_URL=http://vllm-host:8000/v1 +``` + +Replace the URL with the URL of your vLLM service, noting the `v1` suffix. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vllm-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vllm-k8s.md new file mode 100644 index 00000000..2168b378 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/model/vllm-k8s.md @@ -0,0 +1,9 @@ +The vLLM service URL must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} \ + create secret generic vllm-credentials \ + --from-literal=vllm-url=http://vllm:8000/v1 +``` + +Replace the URL with the URL of your vLLM service. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/aks.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/aks.md new file mode 100644 index 00000000..4ab0e703 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/aks.md @@ -0,0 +1 @@ +You need to have an Azure account, and a running AKS cluster. You also need to be authenticated with the cluster and be able to see the cluster state. See [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/products/kubernetes-service). diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/docker-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/docker-compose.md new file mode 100644 index 00000000..c7b5afde --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/docker-compose.md @@ -0,0 +1 @@ +You need to have Docker Compose installed. See [Installing Docker Compose](https://docs.docker.com/compose/install/). diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/eks.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/eks.md new file mode 100644 index 00000000..39c55649 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/eks.md @@ -0,0 +1 @@ +You need to have an AWS account, and a running EKS cluster. You also need to be authenticated with the cluster and be able to see the cluster state. See [Amazon Elastic Kubernetes Service](https://aws.amazon.com/eks/). diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/gke.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/gke.md new file mode 100644 index 00000000..12b2e516 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/gke.md @@ -0,0 +1 @@ +You need to have a Google Cloud account, and a running GKE cluster. You also need to be authenticated with the cluster and be able to see the cluster state. See [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine). diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/minikube.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/minikube.md new file mode 100644 index 00000000..a036aea8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/minikube.md @@ -0,0 +1 @@ +You need to have the Minikube cluster installed and running. See [Minikube - Get Started!](https://minikube.sigs.k8s.io/docs/start). There is TrustGraph documentation on Minikube [here](https://trustgraph.ai/docs/running/minikube). diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/ovh.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/ovh.md new file mode 100644 index 00000000..8bd0e1eb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/ovh.md @@ -0,0 +1 @@ +You need to have an OVHcloud account, and a running Managed Kubernetes cluster. You also need to be authenticated with the cluster and be able to see the cluster state. See [OVHcloud Managed Kubernetes](https://www.ovhcloud.com/en/public-cloud/kubernetes/). diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/podman-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/podman-compose.md new file mode 100644 index 00000000..903b73ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/podman-compose.md @@ -0,0 +1 @@ +You need to have the Podman environment and Podman Compose installed. This should be available with your Linux distribution. See [Beginner's Guide to Using Podman Compose](https://linuxhandbook.com/podman-compose/). diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/scw.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/scw.md new file mode 100644 index 00000000..38328a3e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/platform/scw.md @@ -0,0 +1 @@ +You need to have a Scaleway account, and a running Kubernetes Kapsule cluster. You also need to be authenticated with the cluster and be able to see the cluster state. See [Scaleway Kubernetes Kapsule](https://www.scaleway.com/en/kubernetes-kapsule/). diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/storage/falkordb.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/storage/falkordb.md new file mode 100644 index 00000000..fe773d4f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/storage/falkordb.md @@ -0,0 +1,3 @@ +FalkorDB is licensed under the [Server Side Public License (SSPLv1)](https://github.com/FalkorDB/FalkorDB/blob/master/LICENSE.txt). + +> "The Server Side Public License (SSPLv1) is designed to ensure that if you use FalkorDB as part of a service you make available to others (e.g., in the cloud or as an API), you are required to make the source code of your complete service available under the SSPLv1 license. This is similar to GPL but extends to server use." diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/storage/pinecone-compose.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/storage/pinecone-compose.md new file mode 100644 index 00000000..41be8b0b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/storage/pinecone-compose.md @@ -0,0 +1,5 @@ +To use Pinecone, you need an API token which must be provided in an environment variable. The API token can be created in the Pinecone console of your account. + +``` +PINECONE_API_KEY=TOKEN-GOES-HERE +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/storage/pinecone-k8s.md b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/storage/pinecone-k8s.md new file mode 100644 index 00000000..3a99ddd5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/docs/storage/pinecone-k8s.md @@ -0,0 +1,7 @@ +To use Pinecone, you need an API token which must be provided in a Kubernetes secret. + +```bash +kubectl -n {{namespace}} create secret \ + generic pinecone-api-key \ + --from-literal=pinecone-api-key=PINECONE-API-KEY +``` diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/trustgraph-docs.yaml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/trustgraph-docs.yaml new file mode 100644 index 00000000..2a419ba2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/trustgraph-docs.yaml @@ -0,0 +1,431 @@ +# TrustGraph Documentation Manifest +# +# Maps configuration state to documentation fragments. +# Assembler evaluates 'when' conditions (JSONata) against wizard state, +# selects matching instructions, groups by category, orders by priority. +# +# Output formats: +# - Web/checklist: Structured JSON with completion tracking +# - CLI: Concatenated README.md + +documentation: + id: "trustgraph-deployment-guide" + title: "TrustGraph Deployment Guide" + version: "1.0" + + # Categories define grouping and display order + categories: + - id: platform + title: "Platform Setup" + priority: 1 + - id: model + title: "Model Configuration" + priority: 2 + - id: storage + title: "Storage Setup" + priority: 3 + - id: gateway + title: "API Gateway" + priority: 4 + - id: deployment + title: "Deployment" + priority: 5 + - id: verification + title: "Verification & Testing" + priority: 6 + + instructions: + + # ───────────────────────────────────────────────────────────── + # Platform Setup + # ───────────────────────────────────────────────────────────── + - id: platform-setup + goal: "Install and configure Docker Compose" + category: platform + priority: 1 + file: platform/docker-compose.md + when: "platform = 'docker-compose'" + + - id: platform-setup + goal: "Install and configure Podman Compose" + category: platform + priority: 1 + file: platform/podman-compose.md + when: "platform = 'podman-compose'" + + - id: platform-setup + goal: "Install and configure Minikube" + category: platform + priority: 1 + file: platform/minikube.md + when: "platform = 'minikube'" + + - id: platform-setup + goal: "Configure Google Kubernetes Engine" + category: platform + priority: 1 + file: platform/gke.md + when: "platform = 'gke'" + + - id: platform-setup + goal: "Configure AWS EKS" + category: platform + priority: 1 + file: platform/eks.md + when: "platform = 'eks'" + + - id: platform-setup + goal: "Configure Azure AKS" + category: platform + priority: 1 + file: platform/aks.md + when: "platform = 'aks'" + + - id: platform-setup + goal: "Configure Scaleway Kubernetes" + category: platform + priority: 1 + file: platform/scw.md + when: "platform = 'scw'" + + - id: platform-setup + goal: "Configure OVHcloud Kubernetes" + category: platform + priority: 1 + file: platform/ovh.md + when: "platform = 'ovh'" + + # ───────────────────────────────────────────────────────────── + # Model Configuration - Compose variants + # ───────────────────────────────────────────────────────────── + - id: model-setup + goal: "Set up Ollama model server" + category: model + priority: 1 + file: model/ollama-compose.md + when: "model_deployment = 'ollama' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Configure OpenAI integration" + category: model + priority: 1 + file: model/openai-compose.md + when: "model_deployment = 'openai' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Configure Claude (Anthropic) integration" + category: model + priority: 1 + file: model/claude-compose.md + when: "model_deployment = 'claude' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Configure Mistral integration" + category: model + priority: 1 + file: model/mistral-compose.md + when: "model_deployment = 'mistral' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Configure Cohere integration" + category: model + priority: 1 + file: model/cohere-compose.md + when: "model_deployment = 'cohere' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Configure Azure AI integration" + category: model + priority: 1 + file: model/azure-compose.md + when: "model_deployment = 'azure' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Configure Azure OpenAI integration" + category: model + priority: 1 + file: model/azure-openai-compose.md + when: "model_deployment = 'azure-openai' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Configure Amazon Bedrock integration" + category: model + priority: 1 + file: model/bedrock-compose.md + when: "model_deployment = 'bedrock' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Configure Google AI Studio integration" + category: model + priority: 1 + file: model/google-ai-studio-compose.md + when: "model_deployment = 'google-ai-studio' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Configure Vertex AI integration" + category: model + priority: 1 + file: model/vertex-ai-compose.md + when: "model_deployment = 'vertex-ai' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Set up Llamafile" + category: model + priority: 1 + file: model/llamafile-compose.md + when: "model_deployment = 'llamafile' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Set up LM Studio" + category: model + priority: 1 + file: model/lm-studio-compose.md + when: "model_deployment = 'lm-studio' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Set up vLLM" + category: model + priority: 1 + file: model/vllm-compose.md + when: "model_deployment = 'vllm' and platform in ['docker-compose', 'podman-compose']" + + - id: model-setup + goal: "Set up Text Generation Inference" + category: model + priority: 1 + file: model/tgi-compose.md + when: "model_deployment = 'tgi' and platform in ['docker-compose', 'podman-compose']" + + # ───────────────────────────────────────────────────────────── + # Model Configuration - Kubernetes variants + # ───────────────────────────────────────────────────────────── + - id: model-setup + goal: "Set up Ollama model server" + category: model + priority: 1 + file: model/ollama-k8s.md + when: "model_deployment = 'ollama' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Configure OpenAI integration" + category: model + priority: 1 + file: model/openai-k8s.md + when: "model_deployment = 'openai' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Configure Claude (Anthropic) integration" + category: model + priority: 1 + file: model/claude-k8s.md + when: "model_deployment = 'claude' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Configure Mistral integration" + category: model + priority: 1 + file: model/mistral-k8s.md + when: "model_deployment = 'mistral' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Configure Cohere integration" + category: model + priority: 1 + file: model/cohere-k8s.md + when: "model_deployment = 'cohere' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Configure Azure AI integration" + category: model + priority: 1 + file: model/azure-k8s.md + when: "model_deployment = 'azure' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Configure Azure OpenAI integration" + category: model + priority: 1 + file: model/azure-openai-k8s.md + when: "model_deployment = 'azure-openai' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Configure Amazon Bedrock integration" + category: model + priority: 1 + file: model/bedrock-k8s.md + when: "model_deployment = 'bedrock' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Configure Google AI Studio integration" + category: model + priority: 1 + file: model/google-ai-studio-k8s.md + when: "model_deployment = 'google-ai-studio' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Configure Vertex AI integration" + category: model + priority: 1 + file: model/vertex-ai-k8s.md + when: "model_deployment = 'vertex-ai' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Set up Llamafile" + category: model + priority: 1 + file: model/llamafile-k8s.md + when: "model_deployment = 'llamafile' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Set up LM Studio" + category: model + priority: 1 + file: model/lm-studio-k8s.md + when: "model_deployment = 'lm-studio' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Set up vLLM" + category: model + priority: 1 + file: model/vllm-k8s.md + when: "model_deployment = 'vllm' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: model-setup + goal: "Set up Text Generation Inference" + category: model + priority: 1 + file: model/tgi-k8s.md + when: "model_deployment = 'tgi' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + # ───────────────────────────────────────────────────────────── + # Storage Setup + # ───────────────────────────────────────────────────────────── + - id: vector-db-setup + goal: "Configure Pinecone connection" + category: storage + priority: 1 + file: storage/pinecone-compose.md + when: "vector_db = 'pinecone' and platform in ['docker-compose', 'podman-compose']" + + - id: vector-db-setup + goal: "Configure Pinecone connection" + category: storage + priority: 1 + file: storage/pinecone-k8s.md + when: "vector_db = 'pinecone' and platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + - id: graph-store-setup + goal: "Understand FalkorDB license" + category: storage + priority: 2 + file: storage/falkordb.md + when: "graph_store = 'falkordb'" + + # ───────────────────────────────────────────────────────────── + # API Gateway + # ───────────────────────────────────────────────────────────── + - id: gateway-setup + goal: "Configure API gateway secret" + category: gateway + priority: 1 + file: gateway/gateway-compose.md + when: "platform in ['docker-compose', 'podman-compose']" + + - id: gateway-setup + goal: "Configure API gateway secret" + category: gateway + priority: 1 + file: gateway/gateway-k8s.md + when: "platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + # ───────────────────────────────────────────────────────────── + # MCP Server + # ───────────────────────────────────────────────────────────── + - id: mcp-setup + goal: "Configure MCP server secrets" + category: gateway + priority: 2 + file: gateway/mcp-compose.md + when: "platform in ['docker-compose', 'podman-compose']" + + - id: mcp-setup + goal: "Configure MCP server secrets" + category: gateway + priority: 2 + file: gateway/mcp-k8s.md + when: "platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + vars: + namespace: "k8s.namespace" + + # ───────────────────────────────────────────────────────────── + # Deployment + # ───────────────────────────────────────────────────────────── + - id: deploy + goal: "Deploy with Docker Compose" + category: deployment + priority: 1 + file: deploy/docker-compose-deploy.md + when: "platform = 'docker-compose'" + + - id: deploy + goal: "Deploy with Podman Compose" + category: deployment + priority: 1 + file: deploy/podman-compose-deploy.md + when: "platform = 'podman-compose'" + + - id: deploy + goal: "Deploy to Kubernetes cluster" + category: deployment + priority: 1 + file: deploy/k8s-deploy.md + when: "platform in ['gke', 'eks', 'aks', 'minikube', 'scw', 'ovh']" + + # ───────────────────────────────────────────────────────────── + # Verification & Testing + # ───────────────────────────────────────────────────────────── + - id: workbench + goal: "Access the TrustGraph Workbench" + category: verification + priority: 1 + file: features/workbench.md + always: true + + - id: document-rag + goal: "Test Document RAG" + category: verification + priority: 2 + file: features/document-rag.md + always: true diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/trustgraph-flow.yaml b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/trustgraph-flow.yaml new file mode 100644 index 00000000..9d12c6d5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/trustgraph-flow.yaml @@ -0,0 +1,343 @@ +# TrustGraph Configuration Builder - Dialog Flow (State Machine) +# Each step declares its own transitions to the next state(s) + +flow: + id: "trustgraph-config" + title: "TrustGraph Configuration Builder" + version: "1.0" + start: version + +steps: + + # ───────────────────────────────────────────────────────────── + # Version Selection + # ───────────────────────────────────────────────────────────── + version: + title: "Which TrustGraph version?" + state_key: version + input: + type: select + options: + - label: "TrustGraph 2.2" + value: "2.2" + description: "Additional document format support" + badge: pre-release + recommended: false + - label: "TrustGraph 2.1" + value: "2.1" + description: "Explainability and provenance support" + badge: stable + recommended: true + - label: "TrustGraph 1.9" + value: "1.9" + description: "Improvements to ontology and knowledge extraction prompts" + badge: pre-release + recommended: false + - label: "TrustGraph 1.8" + value: "1.8" + description: "Pluggable messaging fabric, switched to Garage for object store" + badge: stable + recommended: false + transitions: + - next: platform + + # ───────────────────────────────────────────────────────────── + # Platform Selection + # ───────────────────────────────────────────────────────────── + platform: + title: "Which platform?" + description: "Choose how you'll run the TrustGraph containers" + state_key: platform + input: + type: select + options: + - label: "Docker Compose" + value: "docker-compose" + icon: "docker" + description: "Easy to install on MacOS and Linux. Great for evaluation and learning. May not be well suited to production deployments." + recommended: true + - label: "Podman Compose" + value: "podman-compose" + icon: "podman" + description: "Feature-compatible with Docker Compose, default for most Linux systems. May not be well suited to production deployments." + - label: "Minikube" + value: "minikube" + icon: "minikube" + description: "Stand-alone Kubernetes for small-to-medium deployments. Good for learning Kubernetes. Single node cluster limits production use." + - label: "Google Kubernetes Engine (GKE)" + value: "gke" + icon: "google-cloud" + description: "Managed Kubernetes on Google Cloud. Production-ready with full observability and scalability." + - label: "AWS EKS" + value: "eks" + icon: "aws" + description: "Elastic Kubernetes Service on AWS infrastructure. Suitable for production deployment." + - label: "Azure AKS" + value: "aks" + icon: "azure" + description: "Azure Kubernetes Service on Microsoft infrastructure. Suitable for production deployment." + - label: "Scaleway Kubernetes" + value: "scw" + icon: "scaleway" + description: "Managed Kubernetes on Scaleway European cloud infrastructure. Suitable for production deployment." + - label: "OVHCloud Kubernetes" + value: "ovh" + icon: "ovhcloud" + description: "Managed Kubernetes on OVHCloud European infrastructure. Suitable for production deployment." + transitions: + - next: graph-store + + # ───────────────────────────────────────────────────────────── + # Graph Store + # ───────────────────────────────────────────────────────────── + graph-store: + title: "Which graph store?" + description: "Where TrustGraph stores the knowledge graph" + state_key: graph_store + input: + type: select + options: + - label: "Apache Cassandra" + value: "cassandra" + icon: "cassandra" + description: "NoSQL database with fast read/write at scale. TrustGraph integrates a graph schema." + recommended: true + # - label: "Neo4j" + # value: "neo4j" + # icon: "neo4j" + # description: "Native graph database with Cypher query language" + # - label: "Memgraph" + # value: "memgraph" + # icon: "memgraph" + # description: "In-memory graph database with Cypher query language. High performance for real-time analytics." + # - label: "FalkorDB" + # value: "falkordb" + # icon: "falkordb" + # description: "High-performance graph database optimized for low-latency queries. Based on GraphBLAS." + transitions: + - next: vector-db + + # ───────────────────────────────────────────────────────────── + # Vector Database + # ───────────────────────────────────────────────────────────── + vector-db: + title: "Which vector database?" + description: "Stores embeddings for semantic search and RAG" + state_key: vector_db + input: + type: select + options: + - label: "Qdrant" + value: "qdrant" + icon: "qdrant" + description: "High RPS, minimal latency, fast indexing with accuracy control" + recommended: true + # - label: "Milvus" + # value: "milvus" + # icon: "milvus" + # description: "Open-source vector database built for scale" + # - label: "Pinecone" + # value: "pinecone" + # icon: "pinecone" + # description: "Managed vector database service (requires API key)" + transitions: + - next: row-store + + # ───────────────────────────────────────────────────────────── + # Row Store + # ───────────────────────────────────────────────────────────── + row-store: + title: "Which row data store?" + description: "Storage for row/tabular data" + state_key: row_store + input: + type: select + options: + - label: "Apache Cassandra" + value: "cassandra" + icon: "cassandra" + description: "NoSQL database with fast read/write performance at scale. Used for row storage in TrustGraph." + transitions: + - next: model-deployment + + # ───────────────────────────────────────────────────────────── + # Model Deployment + # ───────────────────────────────────────────────────────────── + model-deployment: + title: "How will you run the LLM?" + description: "Choose where the language model runs" + state_key: model_deployment + input: + type: select + options: + # Local / Self-hosted + - label: "Ollama" + value: "ollama" + icon: "ollama" + description: "Run LLMs locally. Bundles model weights, configs, and data into a single package. No external API dependencies." + - label: "Llamafile" + value: "llamafile" + icon: "llamafile" + description: "Single-file LLM distribution with embedded inference server. Runs on most computers with no dependencies." + - label: "LM Studio" + value: "lm-studio" + icon: "lm-studio" + description: "Easy local LLM running. Very usable, great support. Commercial, free for personal use." + # Cloud API Providers + - label: "OpenAI" + value: "openai" + icon: "openai" + description: "Industry-leading models (GPT-4, etc). Requires API key." + - label: "Claude (Anthropic)" + value: "claude" + icon: "anthropic" + description: "Helpful, honest, and harmless AI models. Designed to be trustworthy and reliable. Requires API key." + - label: "Mistral" + value: "mistral" + icon: "mistral" + description: "Efficient models excelling in multilingual tasks, code generation, and reasoning. Strong performance relative to size. Requires API key." + - label: "Cohere" + value: "cohere" + icon: "cohere" + description: "Cloud-agnostic AI with high security, privacy, and customization. Supports on-premises and private cloud. Requires API key." + # Cloud Platform Services + - label: "Azure AI" + value: "azure" + icon: "azure" + description: "Azure Endpoint Services for building, deploying, and managing AI applications. Requires Azure subscription." + - label: "Azure OpenAI" + value: "azure-openai" + icon: "azure" + description: "OpenAI models hosted on Azure infrastructure. Requires Azure subscription." + - label: "Amazon Bedrock" + value: "bedrock" + icon: "aws" + description: "Fully managed service for generative AI on AWS. Multiple model providers available. Requires AWS account." + - label: "Google AI Studio" + value: "google-ai-studio" + icon: "google" + description: "Integrated environment to prototype and experiment with Google's generative AI models. Requires Google account." + - label: "Vertex AI" + value: "vertex-ai" + icon: "google-cloud" + description: "Google Cloud platform for training and deploying ML models. Requires GCP account." + # Self-hosted inference servers + - label: "vLLM" + value: "vllm" + icon: "vllm" + description: "High-throughput LLM serving engine. Self-hosted with OpenAI-compatible API." + - label: "Text Generation Inference (TGI)" + value: "tgi" + icon: "huggingface" + description: "Hugging Face's production-ready inference server. Self-hosted with high performance." + transitions: + - next: max-output-tokens + + # ───────────────────────────────────────────────────────────── + # Max Output Tokens + # ───────────────────────────────────────────────────────────── + max-output-tokens: + title: "Maximum output tokens?" + description: "Limits the length of LLM responses" + state_key: max_output_tokens + input: + type: number + default: 4096 + min: 256 + max: 65536 + step: 256 + transitions: + - next: ocr-enabled + + # ───────────────────────────────────────────────────────────── + # OCR Pipelines Toggle + # ───────────────────────────────────────────────────────────── + ocr-enabled: + title: "Enable OCR processing?" + description: "Replace standard PDF decoding with Optical Character Recognition for scanned documents and images" + state_key: ocr.enabled + input: + type: toggle + default: false + transitions: + - when: "ocr.enabled = true" + next: ocr-engine + - next: embeddings-enabled + + # ───────────────────────────────────────────────────────────── + # OCR Engine (conditional) + # ───────────────────────────────────────────────────────────── + ocr-engine: + title: "Which OCR engine?" + state_key: ocr.engine + input: + type: select + options: + - label: "PDF Decode" + value: "pdf-decode" + description: "Default configuration. Extracts text from PDFs with structured text, but does not perform OCR on images or scanned documents." + recommended: true + - label: "Tesseract" + value: "tesseract" + description: "Free, open-source embedded OCR engine. Best-in-class for free/open-source. Use with PDF documents containing image scans." + - label: "Mistral" + value: "mistral" + description: "Best-in-class commercial OCR service. Requires a Mistral subscription. Use with PDF documents containing image scans." + transitions: + - next: embeddings-enabled + + # ───────────────────────────────────────────────────────────── + # Embeddings Configuration Toggle + # ───────────────────────────────────────────────────────────── + embeddings-enabled: + title: "Configure embeddings engine?" + description: "Customize the embeddings model used for semantic search" + state_key: embeddings.enabled + input: + type: toggle + default: false + transitions: + # Version gate: embeddings config not available before 1.6.0 + - when: "version < '1.6.0'" + next: review + - when: "embeddings.enabled = true" + next: embeddings-engine + - next: review + + # ───────────────────────────────────────────────────────────── + # Embeddings Engine (conditional) + # ───────────────────────────────────────────────────────────── + embeddings-engine: + title: "Which embeddings engine?" + state_key: embeddings.engine + input: + type: select + options: + - label: "FastEmbed" + value: "fastembed" + icon: "fastembed" + description: "Lightweight, fast Python library for embedding generation. Small container image and quick start time." + recommended: true + - label: "HuggingFace sentence-transformers" + value: "huggingface-sentence-transformers" + icon: "huggingface" + description: "Support for a large number of open/community models. Use for non-standard models. Larger container image and longer start time due to PyTorch dependencies." + - label: "Ollama" + value: "ollama" + icon: "ollama" + description: "Use Ollama for embeddings. Requires Ollama to be running with an embedding model loaded." + transitions: + - next: review + + # ───────────────────────────────────────────────────────────── + # Review & Generate (terminal) + # ───────────────────────────────────────────────────────────── + review: + title: "Review your configuration" + type: review + actions: + - id: generate + label: "Generate" + icon: "rocket" + description: "Generate the configuration package as a ZIP file" + transitions: [] # terminal state diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/trustgraph-output.jsonata b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/trustgraph-output.jsonata new file mode 100644 index 00000000..402826aa --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/resources/dialog/trustgraph-output.jsonata @@ -0,0 +1,175 @@ +( + /* + * TrustGraph Configuration Output Transform + * + * Input: State object collected from wizard + * Output: Configuration object with version, platform, and templates array + * + * Example input: + * { + * "version": "1.8.18", + * "platform": "docker-compose", + * "graph_store": "cassandra", + * "vector_db": "qdrant", + * "row_store": "cassandra", + * "model_deployment": "ollama", + * "max_output_tokens": 2048, + * "ocr": { "enabled": true, "engine": "tesseract" }, + * "embeddings": { "enabled": true, "engine": "fastembed" }, + * "k8s": { "namespace": "trustgraph" } + * } + */ + + /* ───────────────────────────────────────────────────────────── + * Helper: Build a component object + * ───────────────────────────────────────────────────────────── */ + $component := function($name, $params) { + { + "name": $name, + "parameters": $params ? $params : {} + } + }; + + /* ───────────────────────────────────────────────────────────── + * Core Infrastructure Components + * ───────────────────────────────────────────────────────────── */ + $infrastructure := [ + /* Graph store */ + $component("triple-store-" & graph_store), + + /* Row store */ + $component("row-store-" & row_store), + + /* Vector database */ + $component("vector-store-" & vector_db), + + /* Messaging - always included */ + $component("pulsar"), + + /* Monitoring - always included */ + $component("grafana") + ]; + + + /* ───────────────────────────────────────────────────────────── + * Model Deployment Components + * ───────────────────────────────────────────────────────────── */ + $model_components := ( + /* Map deployment choice to component name */ + $deployment_map := { + "ollama": "ollama", + "llamafile": "llamafile", + "lm-studio": "lmstudio", + "openai": "openai", + "claude": "claude", + "mistral": "mistral", + "cohere": "cohere", + "azure": "azure", + "azure-openai": "azure-openai", + "bedrock": "bedrock", + "google-ai-studio": "googleaistudio", + "vertex-ai": "vertexai", + "vllm": "vllm", + "tgi": "tgi" + }; + + $component_name := $lookup($deployment_map, model_deployment); + + [ + $component($component_name, { + "max-output-tokens": max_output_tokens + }) + ] + ); + + /* ───────────────────────────────────────────────────────────── + * OCR Components (conditional) + * ───────────────────────────────────────────────────────────── */ + $ocr_map := { + "tesseract": "ocr", + "mistral": "mistral-ocr" + }; + + $ocr_components := ( + ocr.enabled and ocr.engine != "pdf-decode" + ? [ + $component($lookup($ocr_map, ocr.engine)) + ] + : [] + ); + + /* ───────────────────────────────────────────────────────────── + * Embeddings Components (conditional) + * ───────────────────────────────────────────────────────────── */ + $embeddings_map := { + "fastembed": "fastembed", + "huggingface-sentence-transformers": "hf", + "ollama": "ollama" + }; + + $embeddings_components := ( + embeddings.enabled + ? [ + $component("embeddings-" & $lookup($embeddings_map, embeddings.engine)) + ] + : [ + /* Default embeddings if not customized */ + $component("embeddings-fastembed") + ] + ); + + /* ───────────────────────────────────────────────────────────── + * Base Configuration + * ───────────────────────────────────────────────────────────── */ + $base_components := [ + $component("trustgraph-base") + ]; + + /* ───────────────────────────────────────────────────────────── + * Assemble Templates Array + * ───────────────────────────────────────────────────────────── */ + $templates := $append( + $append( + $append( + $append( + $base_components, + $infrastructure + ), + $model_components + ), + $ocr_components + ), + $embeddings_components + ); + + /* ───────────────────────────────────────────────────────────── + * Platform Mapping (dialog values to API values) + * ───────────────────────────────────────────────────────────── */ + $platform_map := { + "docker-compose": "docker-compose", + "podman-compose": "podman-compose", + "minikube": "minikube-k8s", + "gke": "gcp-k8s", + "eks": "eks-k8s", + "aks": "aks-k8s", + "scw": "scw-k8s", + "ovh": "ovh-k8s" + }; + $api_platform := $lookup($platform_map, platform); + + /* ───────────────────────────────────────────────────────────── + * API Configuration + * ───────────────────────────────────────────────────────────── */ + $api_base := "https://config-ui.demo.trustgraph.ai/api/generate"; + $api_url := $api_base & "/" & $api_platform & "/" & version; + + /* ───────────────────────────────────────────────────────────── + * Final Output + * ───────────────────────────────────────────────────────────── */ + { + "version": version, + "platform": $api_platform, + "api_url": $api_url, + "templates": $templates + } +) diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/run.py b/ai-context/trustgraph-templates/trustgraph_configurator/run.py new file mode 100644 index 00000000..544f165d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/run.py @@ -0,0 +1,107 @@ + +import json +import logging +import argparse +import sys + +from . import Generator, Packager + +def run(): + + parser = argparse.ArgumentParser( + prog="tg-build-deployment", + description=__doc__ + ) + + parser.add_argument( + '-v', '--version', + help=f'Version' + ) + + parser.add_argument( + '-i', '--input', + default="config.json", + help=f'Input configuration name (default: config.json)' + ) + + parser.add_argument( + '-o', '--output', + default="deploy.zip", + help=f'Output file name (default: deploy.zip)' + ) + + parser.add_argument( + '-t', '--template', + help=f'Template to use' + ) + + parser.add_argument( + '-p', '--platform', + default="docker-compose", + help=f'Platform (default: docker-compose)' + ) + + parser.add_argument( + '--latest', + action='store_true', + help="Latest version", + ) + + parser.add_argument( + '--latest-stable', + action='store_true', + help="Latest stable version", + ) + + parser.add_argument( + '-O', '--output-tg-config', + action='store_true', + help="Output only TrustGraph configuration to stdout", + ) + + parser.add_argument( + '-R', '--output-resources', + action='store_true', + help="Output only resources (docker-compose.yaml or resources.yaml) to stdout", + ) + + try: + + args = parser.parse_args() + args = vars(args) + + input = args["input"] + + with open(input) as f: + config = f.read() + + output = args["output"] + output_tg_config = args.get("output_tg_config", False) + output_resources = args.get("output_resources", False) + + # Configure logging only if not outputting to stdout + if not output_tg_config and not output_resources: + logging.basicConfig(level=logging.INFO, format='%(message)s') + else: + # Suppress all logging when outputting to stdout + logging.basicConfig(level=logging.CRITICAL) + + del args["input"] + del args["output"] + del args["output_tg_config"] + del args["output_resources"] + + a = Packager(**args) + + if output_tg_config: + a.write_tg_config(config) + elif output_resources: + a.write_resources(config) + else: + a.write(config, output) + + except Exception as e: + + print(f"Exception: {e}", file=sys.stderr) + sys.exit(1) + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/service.py b/ai-context/trustgraph-templates/trustgraph_configurator/service.py new file mode 100644 index 00000000..f5b8d6be --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/service.py @@ -0,0 +1,18 @@ + +import logging + +from . api import Api + +def run_service(): + + logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s %(levelname)s %(message)s" + ) + + logging.info("Starting...") + + a = Api(port=8080) + + a.run() + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/README.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/README.md new file mode 100644 index 00000000..23039e9a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/README.md @@ -0,0 +1,125 @@ + +# TrustGraph template generation + +There are two utilities here: + +- `generate`: Generates a single Docker Compose launch configuration + based on configuration you provide. +- `generate-all`: Generates the release bundle for releases. You won't + need to use this unless you are managing releases. + +## `generate-all` + +Previously, this generates a full set of all vector DB / triple store / LLM +combinations, and put them in a single ZIP file. But this got out of +hand, so at the time of writing, this generates a single configuraton +using Qdrant vector DB, Ollama LLM support and Cassandra for a triple store. + +The combinations are contained withing the code, it takes two arguments: +- output ZIP file (is over-written) +- TrustGraph version number + +``` +templates/generate-all output.zip 0.18.11 +``` + +## `generate` + +This utility takes a configuration file describing the components to bundle, +and outputs a Docker Compose YAML file. + +### Input configuration + +The input configuration is a JSON file, an array of components to pull into +the configuration. For each component, there is a name and a (possibly empty) +object describing addtional parameters for that component. + +Example: + +``` +[ + { + "name": "cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "qdrant", + "parameters": {} + }, + { + "name": "embeddings-hf", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph", + "parameters": {} + }, + { + "name": "googleaistudio", + "parameters": { + "googleaistudio-temperature": 0.3, + "googleaistudio-max-output-tokens": 2048, + "googleaistudio-model": "gemini-1.5-pro-002" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 1000, + "chunk-overlap": 50 + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "agent-manager-react", + "parameters": {} + } +] +``` + +If you want to make your own configuration you could try changing the +configuration above: +- Components which are essential: pulsar, trustgraph, graph-rag, grafana, + agent-manager-react +- You need a triple store, one of: cassandra, memgraph, falkordb, neo4j +- You need a vector store, one of: qdrant, pinecone +- You need an LLM, one of: azure, azure-openai, bedrock, claude, cohere, + llamafile, ollama, openai, vertexai. +- You need an embeddings implementation, one of: embeddings-hf, + embeddings-ollama +- Optionally add the Workbench tool: workbench-ui + +Components have over-ridable parameters, look in the component definition +in `templates/components/` to see what you can override. + +### Invocation + +Two parameters: +- The output ZIP file +- The version number + +The configuration file described above is provided on standard input + +``` +templates/generate out.zip 0.18.9 < config.json +``` + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/base/base.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/base/base.jsonnet new file mode 100644 index 00000000..9f82efb2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/base/base.jsonnet @@ -0,0 +1,3 @@ +{ + restart: "on-failure:100", +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components.jsonnet new file mode 100644 index 00000000..41391a9e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components.jsonnet @@ -0,0 +1,98 @@ +{ + + // Essentials + "trustgraph-base": import "components/trustgraph.jsonnet", + "rev-gateway": import "components/rev-gateway.jsonnet", + "pulsar": import "components/pulsar.jsonnet", + + // LLMs + "azure": import "components/azure.jsonnet", + "azure-openai": import "components/azure-openai.jsonnet", + "bedrock": import "components/bedrock.jsonnet", + "claude": import "components/claude.jsonnet", + "cohere": import "components/cohere.jsonnet", + "googleaistudio": import "components/googleaistudio.jsonnet", + "llamafile": import "components/llamafile.jsonnet", + "lmstudio": import "components/lmstudio.jsonnet", + "mistral": import "components/mistral.jsonnet", + "ollama": import "components/ollama.jsonnet", + "openai": import "components/openai.jsonnet", + "vertexai": import "components/vertexai.jsonnet", + "tgi": import "components/tgi.jsonnet", + "vllm": import "components/vllm.jsonnet", + + // LLMs for RAG. RAG components have been collapsed into the core + // component, so gone away. + "azure-rag": {}, + "azure-openai-rag": {}, + "bedrock-rag": {}, + "claude-rag": {}, + "cohere-rag": {}, + "googleaistudio-rag": {}, + "llamafile-rag": {}, + "lmstudio-rag": {}, + "mistral-rag": {}, + "ollama-rag": {}, + "openai-rag": {}, + "vertexai-rag": {}, + "tgi-rag": import "components/tgi-rag.jsonnet", + "vllm-rag": {}, + + "tgi-service-cpu": import "components/tgi-service-cpu.jsonnet", + "tgi-service-intel-gpu": import "components/tgi-service-intel-gpu.jsonnet", + "tgi-service-gaudi": import "components/tgi-service-gaudi.jsonnet", + + "vllm-service-intel-gpu": import "components/vllm-service-intel-gpu.jsonnet", + "vllm-service-gaudi": import "components/vllm-service-gaudi.jsonnet", + "vllm-service-nvidia": import "components/vllm-service-nvidia.jsonnet", + + // Embeddings + "embeddings-ollama": import "components/embeddings-ollama.jsonnet", + "embeddings-hf": import "components/embeddings-hf.jsonnet", + "embeddings-fastembed": import "components/embeddings-fastembed.jsonnet", + + // OCR options + "ocr": import "components/ocr.jsonnet", + "mistral-ocr": import "components/mistral-ocr.jsonnet", + + // Vector stores + "vector-store-milvus": import "components/vector-store-milvus.jsonnet", + "vector-store-qdrant": import "components/vector-store-qdrant.jsonnet", + "vector-store-pinecone": import "components/vector-store-pinecone.jsonnet", + + // Triples stores + "triple-store-cassandra": import "components/triple-store-cassandra.jsonnet", + "triple-store-neo4j": import "components/triple-store-neo4j.jsonnet", + "triple-store-falkordb": import "components/triple-store-falkordb.jsonnet", + "triple-store-memgraph": import "components/triple-store-memgraph.jsonnet", + + // Object stores + "row-store-cassandra": import "components/row-store-cassandra.jsonnet", + + // Observability support + "grafana": import "components/grafana.jsonnet", + + // Pulsar manager is a UI for Pulsar. Uses a LOT of memory + "pulsar-manager": import "components/pulsar-manager.jsonnet", + + "override-recursive-chunker": import "components/chunker-recursive.jsonnet", + + // The prompt manager + "prompt-overrides": import "components/prompt-overrides.jsonnet", + + // Archaic - part of core system, just making sure these don't + // cause a failure + "workbench-ui": {}, + "prompt-template": {}, + "agent-manager-react": {}, + "graph-rag": {}, + "document-rag": {}, + "librarian": {}, + + // Extra MCP services + "ddg-mcp-server": import "mcp/ddg-mcp-server.jsonnet", + + // Does nothing. But, can be a hack to overwrite parameters + "null": {}, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/agent-manager-react.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/agent-manager-react.jsonnet new file mode 100644 index 00000000..d99ae88c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/agent-manager-react.jsonnet @@ -0,0 +1,40 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "agent-manager" +: { + + create:: function(engine) + + local container = + engine.container("agent-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "agent-manager-react", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "agent-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/azure-openai.jsonnet new file mode 100644 index 00000000..4d5a70e8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/azure-openai.jsonnet @@ -0,0 +1,113 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure-openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-openai-" + key]:: value, + }, + +// Strategy is to specify the model with the AZURE_MODEL environment +// variable. This isn't something that can just be specified dynamically, +// it has to match what was provisioned in Azure. + + "azure-openai-max-output-tokens":: 4192, + "azure-openai-temperature":: 0.0, + "azure-openai-models":: models, + + "llm-models" +:: $["azure-openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/azure.jsonnet new file mode 100644 index 00000000..f4db7500 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/azure.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-" + key]:: value, + }, + + "azure-max-output-tokens":: 4096, + "azure-temperature":: 0.0, + "azure-models":: models, + + "llm-models" +:: $["azure-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/bedrock.jsonnet new file mode 100644 index 00000000..19bf1459 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/bedrock.jsonnet @@ -0,0 +1,106 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local chunker = import "chunker-recursive.jsonnet"; +local models = import "parameters/bedrock.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["bedrock-" + key]:: value, + }, + + "bedrock-max-output-tokens":: 4096, + "bedrock-temperature":: 0.0, + "bedrock-models":: models, + + "llm-models" +:: $["bedrock-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + chunker + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/chunker-recursive.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/chunker-recursive.jsonnet new file mode 100644 index 00000000..a9f31d73 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/chunker-recursive.jsonnet @@ -0,0 +1,48 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "chunk-size":: 2000, + "chunk-overlap":: 100, + + "chunker" +: { + + create:: function(engine) + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + std.toString($["chunk-size"]), + "--chunk-overlap", + std.toString($["chunk-overlap"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/claude.jsonnet new file mode 100644 index 00000000..f2dcc7e1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/claude.jsonnet @@ -0,0 +1,105 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/claude.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["claude-" + key]:: value, + }, + + "claude-max-output-tokens":: 4096, + "claude-temperature":: 0.0, + "claude-models":: models, + + "llm-models" +:: $["claude-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/cohere.jsonnet new file mode 100644 index 00000000..644b7b4a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/cohere.jsonnet @@ -0,0 +1,98 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/cohere.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["cohere-" + key]:: value, + }, + + "cohere-temperature":: 0.0, + "cohere-models":: models, + + "llm-models" +:: $["cohere-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/configuration.jsonnet new file mode 100644 index 00000000..b2499ff6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/configuration.jsonnet @@ -0,0 +1,50 @@ + +// This puts the default configuration together. References many things, +// flow classes, a default flow, token costs, prompts, agent tools + +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "init-trustgraph" +: { + + create:: function(engine) + + local cfgVol = engine.configVolume( + "trustgraph-cfg", "trustgraph", + { + "config.json": importstr "trustgraph/config.json", + } + ); + + local container = + engine.container("init-trustgraph") + .with_image(images.trustgraph_flow) + .with_command( + [ + "tg-init-trustgraph", + "-p", + url.pulsar_admin, + "--config-file", + "/trustgraph/config.json", + ] + ) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M") + .with_volume_mount(cfgVol, "/trustgraph/"); + + local containerSet = engine.containers( + "init-trustgraph", [ container ] + ); + + engine.resources([ + cfgVol, + containerSet, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/document-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/document-rag.jsonnet new file mode 100644 index 00000000..239598b7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/document-rag.jsonnet @@ -0,0 +1,77 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "document-rag-doc-limit":: 20, + + "document-rag" +: { + + create:: function(engine) + + local container = + engine.container("document-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "document-rag", + "-p", + url.pulsar, + "--doc-limit", + std.toString($["document-rag-doc-limit"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "document-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "document-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("document-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "document-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M"); + + local containerSet = engine.containers( + "document-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..1e4d6cb9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/embeddings-fastembed.jsonnet @@ -0,0 +1,48 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-fastembed.jsonnet"; + +{ + + "fastembed-models":: models, + + "embeddings-models" +:: $["fastembed-models"], + + embeddings +: { + + create:: function(engine) + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-fastembed", + "-p", + url.pulsar, + "--concurrency", + std.toString($["embeddings-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/embeddings-hf.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/embeddings-hf.jsonnet new file mode 100644 index 00000000..29008129 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/embeddings-hf.jsonnet @@ -0,0 +1,46 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-huggingface.jsonnet"; + +{ + + "huggingface-embeddings-models":: models, + + "embeddings-models" +:: $["huggingface-embeddings-models"], + + embeddings +: { + + create:: function(engine) + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_hf) + .with_command([ + "embeddings-hf", + "-p", + url.pulsar, + "--concurrency", + std.toString($["embeddings-concurrency"]), + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/embeddings-ollama.jsonnet new file mode 100644 index 00000000..8422b646 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/embeddings-ollama.jsonnet @@ -0,0 +1,51 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local models = import "parameters/embeddings-ollama.jsonnet"; + +{ + + "ollama-url":: "${OLLAMA_HOST}", + + "ollama-models":: models, + + "embeddings-models" +:: $["ollama-models"], + + embeddings +: { + + create:: function(engine) + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString($["embeddings-concurrency"]), + "-r", + $["ollama-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/googleaistudio.jsonnet new file mode 100644 index 00000000..9b6ff6c5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/googleaistudio.jsonnet @@ -0,0 +1,105 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/googleaistudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["googleaistudio-" + key]:: value, + }, + + "googleaistudio-max-output-tokens":: 4096, + "googleaistudio-temperature":: 0.0, + "googleaistudio-models":: models, + + "llm-models" +:: $["googleaistudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/grafana.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/grafana.jsonnet new file mode 100644 index 00000000..e968faec --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/grafana.jsonnet @@ -0,0 +1,122 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "prometheus" +: { + + create:: function(engine) + + local vol = engine.volume("prometheus-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "prometheus-cfg", "prometheus", + { + "prometheus.yml": importstr "prometheus/prometheus.yml", + } + ); + + local container = + engine.container("prometheus") + .with_image(images.prometheus) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M") + .with_port(9090, 9090, "http") + .with_volume_mount(cfgVol, "/etc/prometheus/") + .with_volume_mount(vol, "/prometheus"); + + local containerSet = engine.containers( + "prometheus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9090, 9090, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + + "grafana" +: { + + create:: function(engine) + + local vol = engine.volume("grafana-storage").with_size("20G"); + + local provDashVol = engine.configVolume( + "prov-dash", "grafana/provisioning/", + { + "dashboard.yml": + importstr "grafana/provisioning/dashboard.yml", + } + + ); + + local provDataVol = engine.configVolume( + "prov-data", "grafana/provisioning/", + { + "datasource.yml": + importstr "grafana/provisioning/datasource.yml", + } + + ); + + local dashVol = engine.configVolume( + "dashboards", "grafana/dashboards/", + { + "dashboard.json": + importstr "grafana/dashboards/dashboard.json", + } + + ); + + local container = + engine.container("grafana") + .with_image(images.grafana) + .with_environment({ + // GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin", + // GF_AUTH_ANONYMOUS_ENABLED: "true", + // GF_ORG_ROLE: "Admin", + GF_ORG_NAME: "trustgraph.ai", + // GF_SERVER_ROOT_URL: "https://example.com", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(3000, 3000, "cassandra") + .with_volume_mount(vol, "/var/lib/grafana") + .with_volume_mount( + provDashVol, "/etc/grafana/provisioning/dashboards/" + ) + .with_volume_mount( + provDataVol, "/etc/grafana/provisioning/datasources/" + ) + .with_volume_mount( + dashVol, "/var/lib/grafana/dashboards/" + ); + + local containerSet = engine.containers( + "grafana", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3000, 3000, "http"); + + engine.resources([ + vol, + provDashVol, + provDataVol, + dashVol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/graph-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/graph-rag.jsonnet new file mode 100644 index 00000000..64ab0130 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/graph-rag.jsonnet @@ -0,0 +1,223 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "graph-rag-entity-limit":: 50, + "graph-rag-triple-limit":: 30, + "graph-rag-max-subgraph-size":: 400, + "graph-rag-max-path-length":: 2, + + "kg-extract-definitions" +: { + + create:: function(engine) + + local container = + engine.container("kg-extract-definitions") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-definitions", + "-p", + url.pulsar, + "--concurrency", + std.toString($["kg-extraction-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-extract-definitions", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-relationships" +: { + + create:: function(engine) + + local container = + engine.container("kg-extract-relationships") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-relationships", + "-p", + url.pulsar, + "--concurrency", + std.toString($["kg-extraction-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-extract-relationships", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-agent" +: { + + create:: function(engine) + + local container = + engine.container("kg-extract-agent") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-agent", + "-p", + url.pulsar, + "--concurrency", + std.toString($["kg-extraction-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-extract-agent", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-ontology" +: { + + create:: function(engine) + + local container = + engine.container("kg-extract-ontology") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-ontology", + "-p", + url.pulsar, + "--concurrency", + std.toString($["kg-extraction-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "300M") + .with_reservations("0.1", "300M"); + + local containerSet = engine.containers( + "kg-extract-ontology", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-rag" +: { + + create:: function(engine) + + local container = + engine.container("graph-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-rag", + "-p", + url.pulsar, +// "--concurrency", +// std.toString($["graph-rag-concurrency"]), + "--entity-limit", + std.toString($["graph-rag-entity-limit"]), + "--triple-limit", + std.toString($["graph-rag-triple-limit"]), + "--max-subgraph-size", + std.toString($["graph-rag-max-subgraph-size"]), + "--max-path-length", + std.toString($["graph-rag-max-path-length"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "graph-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M"); + + local containerSet = engine.containers( + "graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/librarian.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/librarian.jsonnet new file mode 100644 index 00000000..cfcbcff0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/librarian.jsonnet @@ -0,0 +1,43 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local minio = import "stores/minio.jsonnet"; +local cassandra = import "stores/cassandra.jsonnet"; + +{ + + "librarian" +: { + + create:: function(engine) + + local container = + engine.container("librarian") + .with_image(images.trustgraph_flow) + .with_command([ + "librarian", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M"); + + local containerSet = engine.containers( + "librarian", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +// Minio and Cassandra are used by the Librarian +} + minio + cassandra + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/llamafile.jsonnet new file mode 100644 index 00000000..bb6c55fd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/llamafile.jsonnet @@ -0,0 +1,95 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/slm.jsonnet"; +local models = import "parameters/llamafile.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["llamafile-" + key]:: value, + }, + + "llamafile-models":: models, + + "llm-models" +:: $["llamafile-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/lmstudio.jsonnet new file mode 100644 index 00000000..a578b694 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/lmstudio.jsonnet @@ -0,0 +1,105 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/lmstudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["lmstudio-" + key]:: value, + }, + + "lmstudio-max-output-tokens":: 4096, + "lmstudio-temperature":: 0.0, + "lmstudio-models":: models, + + "llm-models" +:: $["lmstudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/mcp-server.jsonnet new file mode 100644 index 00000000..7769889e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/mcp-server.jsonnet @@ -0,0 +1,48 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "mcp-server-port":: 8000, + + "mcp-server" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mcp-server-secret") + .with_env_var("MCP_SERVER_SECRET", "mcp-server-secret"); + + local port = $["mcp-server-port"]; + + local container = + engine.container("mcp-server") + .with_image(images.trustgraph_mcp) + .with_command([ + "mcp-server", + "--port", + std.toString(port), + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/mistral-ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/mistral-ocr.jsonnet new file mode 100644 index 00000000..f70e0cd6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/mistral-ocr.jsonnet @@ -0,0 +1,49 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "pdf-decoder" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("mistral-ocr") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-ocr-mistral", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "mistral-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/mistral.jsonnet new file mode 100644 index 00000000..6d01fbc9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/mistral.jsonnet @@ -0,0 +1,105 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/mistral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "mistral-max-output-tokens":: 4096, + "mistral-temperature":: 0.0, + "mistral-models":: models, + + "llm-models" +:: $["mistral-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/null.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/null.jsonnet new file mode 100644 index 00000000..2c63c085 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/null.jsonnet @@ -0,0 +1,2 @@ +{ +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/object-store-cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/object-store-cassandra.jsonnet new file mode 100644 index 00000000..8362b5c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/object-store-cassandra.jsonnet @@ -0,0 +1,78 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "stores/cassandra.jsonnet"; + +cassandra + { + + "store-objects" +: { + + create:: function(engine) + + local container = + engine.container("store-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "objects-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-objects" +: { + + create:: function(engine) + + local container = + engine.container("query-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "objects-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/ocr.jsonnet new file mode 100644 index 00000000..4f53aa5a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/ocr.jsonnet @@ -0,0 +1,37 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "pdf-decoder" +: { + + create:: function(engine) + + local container = + engine.container("pdf-ocr") + .with_image(images.trustgraph_ocr) + .with_command([ + "pdf-ocr", + "-p", + url.pulsar, + ]) + .with_limits("1.0", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "pdf-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/ollama.jsonnet new file mode 100644 index 00000000..bccfd0f1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/ollama.jsonnet @@ -0,0 +1,99 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/ollama.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["ollama-" + key]:: value, + }, + + "ollama-models":: models, + + "llm-models" +:: $["ollama-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString($["text-completion-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString($["text-completion-rag-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/openai.jsonnet new file mode 100644 index 00000000..c8754280 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/openai.jsonnet @@ -0,0 +1,107 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["openai-" + key]:: value, + }, + + "openai-max-output-tokens":: 4096, + "openai-temperature":: 0.0, + "openai-models":: models, + + "llm-models" +:: $["openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/prompt-overrides.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/prompt-overrides.jsonnet new file mode 100644 index 00000000..852ec09d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/prompt-overrides.jsonnet @@ -0,0 +1,24 @@ +local default_prompts = import "prompts/default-prompts.jsonnet"; + +{ + + with:: function(key, value) + if (key == "system-template") then + self + { + prompts +:: { + "system-template": value, + } + } + else + self + { + prompts +:: { + templates +:: { + [key] +:: { + prompt: value + } + } + } + }, + +} + default_prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/prompt-template.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/prompt-template.jsonnet new file mode 100644 index 00000000..5d1af35b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/prompt-template.jsonnet @@ -0,0 +1,78 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "prompt" +: { + + create:: function(engine) + + local container = + engine.container("prompt") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--concurrency", + std.toString($["prompt-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "prompt", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "prompt-rag" +: { + + create:: function(engine) + + local container = + engine.container("prompt-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--id", + "prompt-rag", + "--concurrency", + std.toString($["prompt-rag-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "prompt-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/pulsar-manager.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/pulsar-manager.jsonnet new file mode 100644 index 00000000..9a0b59b2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/pulsar-manager.jsonnet @@ -0,0 +1,41 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "pulsar" +: { + + create:: function(engine) + +// FIXME: Should persist something? +// local volume = engine.volume(...) + + local container = + engine.container("pulsar") + .with_image(images.pulsar_manager) + .with_environment({ + SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties", + }) + .with_limits("0.5", "1.4G") + .with_reservations("0.1", "1.4G") + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2"); + + local containerSet = engine.containers( + "pulsar", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2); + + engine.resources([ + containerSet, + service, + ]) + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/pulsar.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/pulsar.jsonnet new file mode 100644 index 00000000..0cfd1854 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/pulsar.jsonnet @@ -0,0 +1,171 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +// This is a Pulsar configuration. Non-standalone mode so we deploy +// individual components: bookkeeper, broker and zookeeper. +// +// This also deploys the TrustGraph 'admin' container which initialises +// TrustGraph-specific namespaces etc. + +{ + + "pulsar" +: { + + create:: function(engine) + + // Zookeeper volume + local zkVolume = engine.volume("zookeeper").with_size("1G"); + + // Zookeeper container + local zkContainer = + engine.container("zookeeper") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper" + ]) + .with_limits("1", "400M") + .with_reservations("0.05", "400M") + .with_user("0:1000") + .with_volume_mount(zkVolume, "/pulsar/data/zookeeper") + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m", + }) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Pulsar cluster init container + local initContainer = + engine.container("pulsar-init") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650", + ]) + .with_limits("1", "512M") + .with_reservations("0.05", "512M") + .with_environment({ + "PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m", + }); + + + // Bookkeeper volume + local bookieVolume = engine.volume("bookie").with_size("20G"); + + // Bookkeeper container + local bookieContainer = + engine.container("bookie") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + // false ^ causes this to be a 'failure' exit. + ]) + .with_limits("1", "1024M") + .with_reservations("0.1", "1024M") + .with_user("0:1000") + .with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper") + .with_environment({ + "clusterName": "cluster-a", + "zkServers": "zookeeper:2181", + "bookieId": "bookie", + "metadataStoreUri": "metadata-store:zk:zookeeper:2181", + "advertisedAddress": "bookie", + "BOOKIE_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m", + }) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker, stateless (uses ZK and Bookkeeper for state) + local brokerContainer = + engine.container("pulsar") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" + ]) + .with_limits("1", "800M") + .with_reservations("0.1", "800M") + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "zookeeperServers": "zookeeper:2181", + "clusterName": "cluster-a", + "managedLedgerDefaultEnsembleSize": "1", + "managedLedgerDefaultWriteQuorum": "1", + "managedLedgerDefaultAckQuorum": "1", + "advertisedAddress": "pulsar", + "advertisedListeners": "external:pulsar://pulsar:6650,localhost:pulsar://localhost:6650", + "PULSAR_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m", + }) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + // Container sets + local zkContainerSet = engine.containers( + "zookeeper", + [ + zkContainer, + ] + ); + + local initContainerSet = engine.containers( + "init-pulsar", + [ + initContainer, + ] + ); + + local bookieContainerSet = engine.containers( + "bookie", + [ + bookieContainer, + ] + ); + + local brokerContainerSet = engine.containers( + "pulsar", + [ + brokerContainer, + ] + ); + + // Zookeeper service + local zkService = + engine.service(zkContainerSet) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Bookkeeper service + local bookieService = + engine.service(bookieContainerSet) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker service + local brokerService = + engine.service(brokerContainerSet) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + engine.resources([ + zkVolume, + bookieVolume, + zkContainerSet, + initContainerSet, + bookieContainerSet, + brokerContainerSet, + zkService, + bookieService, + brokerService, + ]) + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/rev-gateway.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/rev-gateway.jsonnet new file mode 100644 index 00000000..b549f402 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/rev-gateway.jsonnet @@ -0,0 +1,54 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + // Invalid, but at least means the rev-gateway won't connect to anything + // it shouldn't. + "rev-gateway-token":: "INVALID_TOKEN", + "rev-gateway-uri":: "wss://127.0.0.1/api/v1/relay?token=" + $["rev-gateway-token"], + + "rev-gateway" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("rev-gateway-secret") + .with_env_var("REV_GATEWAY_SECRET", "rev-gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "rev-gateway", + "-p", + url.pulsar, + "--websocket-uri", + std.toString($["rev-gateway-uri"]), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/structured-data.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/structured-data.jsonnet new file mode 100644 index 00000000..33d0ef06 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/structured-data.jsonnet @@ -0,0 +1,137 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "nlp-query" +: { + + create:: function(engine) + + local container = + engine.container("nlp-query") + .with_image(images.trustgraph_flow) + .with_command([ + "nlp-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "nlp-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-query" +: { + + create:: function(engine) + + local container = + engine.container("structured-query") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "structured-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-diag" +: { + + create:: function(engine) + + local container = + engine.container("structured-diag") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-diag", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "96M") + .with_reservations("0.1", "96M"); + + local containerSet = engine.containers( + "structured-diag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-objects" +: { + + create:: function(engine) + + local container = + engine.container("kg-extract-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-objects", + "-p", + url.pulsar, + "--concurrency", + std.toString($["kg-extraction-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-extract-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-rag.jsonnet new file mode 100644 index 00000000..0a8a7af9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-rag.jsonnet @@ -0,0 +1,62 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-rag-" + key]:: value, + }, + + "tgi-rag-max-output-tokens":: 1024, + "tgi-rag-temperature":: 0.0, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString($["text-completion-rag-concurrency"]), + "-x", + std.toString($["tgi-rag-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-rag-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-service-cpu.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-service-cpu.jsonnet new file mode 100644 index 00000000..0a02f210 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-service-cpu.jsonnet @@ -0,0 +1,65 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage").with_size("20G"); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-cpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--cuda-graphs", + "0", + "--port", + "8899" + ]) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_environment({ + HF_TOKEN: $["hf-token"], + }) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(8899, 8899, "tgi") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-service-gaudi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-service-gaudi.jsonnet new file mode 100644 index 00000000..9c52ef3d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-service-gaudi.jsonnet @@ -0,0 +1,92 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "meta-llama/Llama-3.3-70B-Instruct", + "tgi-service-cpus":: "64.0", + "tgi-service-memory":: "64G", + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage").with_size("50G"); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-gaudi"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--sharded", + "true", + "--num-shard", + "8", + "--max-input-tokens", + "4096", + "--max-total-tokens", + "4096", + "--max-batch-size", + "128", +// "--max-batch-prefill-tokens", +// "16384", + "--max-waiting-tokens", + "7", +// "--waiting-served-ratio", +// "1.2", + "--max-concurrent-requests", + "512", + "--cuda-graphs", + "0", + "--port", + "8899" + ]) + .with_runtime("habana") + .with_environment({ + HABANA_VISIBLE_DEVICES: "all", + OMPI_MCA_btl_vader_single_copy_mechanism: "none", + HF_TOKEN: $["hf-token"], + ENABLE_HPU_GRAPH: 'true', + LIMIT_HPU_GRAPH: 'true', + USE_FLASH_ATTENTION: 'true', + FLASH_ATTENTION_RECOMPUTE: 'true', +// PT_HPU_ENABLE_LAZY_COLLECTIVES: 'true', +// PREFILL_BATCH_BUCKET_SIZE: "1", +// BATCH_BUCKET_SIZE: "1", + + }) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(8899, 8899, "tgi") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-service-intel-gpu.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-service-intel-gpu.jsonnet new file mode 100644 index 00000000..ef223288 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi-service-intel-gpu.jsonnet @@ -0,0 +1,65 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage").with_size("20G"); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-intel-xpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--cuda-graphs", + "0", + "--port", + "8899" + ]) + .with_environment({ + HF_TOKEN: $["hf-token"], + }) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(8899, 8899, "tgi") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi.jsonnet new file mode 100644 index 00000000..f62266a2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/tgi.jsonnet @@ -0,0 +1,60 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-" + key]:: value, + }, + + "tgi-max-output-tokens":: 1024, + "tgi-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--concurrency", + std.toString($["text-completion-concurrency"]), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-cassandra.jsonnet new file mode 100644 index 00000000..a7106cb5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-cassandra.jsonnet @@ -0,0 +1,78 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "stores/cassandra.jsonnet"; + +cassandra + { + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-falkordb.jsonnet new file mode 100644 index 00000000..f9177a18 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-falkordb.jsonnet @@ -0,0 +1,80 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local falkordb = import "stores/falkordb.jsonnet"; + +falkordb + { + + "falkordb-url":: "falkor://falkordb:6379", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-memgraph.jsonnet new file mode 100644 index 00000000..bc86a5a7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-memgraph.jsonnet @@ -0,0 +1,85 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local memgraph = import "stores/memgraph.jsonnet"; + +memgraph + { + + "memgraph-url":: "bolt://memgraph:7687", + "memgraph-database":: "memgraph", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-neo4j.jsonnet new file mode 100644 index 00000000..f4a93d9c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/triple-store-neo4j.jsonnet @@ -0,0 +1,80 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local neo4j = import "stores/neo4j.jsonnet"; + +neo4j + { + + "neo4j-url":: "bolt://neo4j:7687", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/trustgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/trustgraph.jsonnet new file mode 100644 index 00000000..1eac5dd8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/trustgraph.jsonnet @@ -0,0 +1,349 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +local config_initialiser = import "configuration.jsonnet"; +local config = import "trustgraph-config.jsonnet"; +local librarian = import "librarian.jsonnet"; +local mcp_server = import "mcp-server.jsonnet"; +local workbench = import "workbench-ui.jsonnet"; +local graphrag = import "graph-rag.jsonnet"; +local documentrag = import "document-rag.jsonnet"; +local prompt_template = import "prompt-template.jsonnet"; +local agent_manager = import "agent-manager-react.jsonnet"; +local structured_data = import "structured-data.jsonnet"; +local ddg = import "mcp/ddg-mcp-server.jsonnet"; + +{ + + "log-level":: "INFO", + + "api-gateway-port":: 8088, + "api-gateway-timeout":: 600, + + "chunk-size":: 250, + "chunk-overlap":: 15, + + "prompt-concurrency":: 1, + "prompt-rag-concurrency":: 1, + + "text-completion-concurrency":: 1, + "text-completion-rag-concurrency":: 1, + + "kg-extraction-concurrency":: 1, + "graph-rag-concurrency":: 1, + + "embeddings-concurrency":: 1, + + "api-gateway" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("gateway-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local port = $["api-gateway-port"]; + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "api-gateway", + "-p", + url.pulsar, + "--timeout", + std.toString($["api-gateway-timeout"]), + "--port", + std.toString(port), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "chunker" +: { + + create:: function(engine) + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-token", + "-p", + url.pulsar, + "--chunk-size", + std.toString($["chunk-size"]), + "--chunk-overlap", + std.toString($["chunk-overlap"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "config-svc" +: { + + create:: function(engine) + + local container = + engine.container("config-svc") + .with_image(images.trustgraph_flow) + .with_command([ + "config-svc", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "config-svc", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "pdf-decoder" +: { + + create:: function(engine) + + local container = + engine.container("pdf-decoder") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-decoder", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "pdf-decoder", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "mcp-tool" +: { + + create:: function(engine) + + local container = + engine.container("mcp-tool") + .with_image(images.trustgraph_flow) + .with_command([ + "mcp-tool", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "mcp-tool", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering" +: { + + create:: function(engine) + + local container = + engine.container("metering") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "metering", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering-rag" +: { + + create:: function(engine) + + local container = + engine.container("metering-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--id", + "metering-rag", + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "metering-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-store" +: { + + create:: function(engine) + + local container = + engine.container("kg-store") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-store", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-store", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-manager" +: { + + create:: function(engine) + + local container = + engine.container("kg-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-manager", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + librarian + mcp_server + workbench + graphrag + + documentrag + prompt_template + agent_manager + structured_data + + config_initialiser + config + + ddg + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vector-store-milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vector-store-milvus.jsonnet new file mode 100644 index 00000000..32d6fab4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vector-store-milvus.jsonnet @@ -0,0 +1,147 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local milvus = import "stores/milvus.jsonnet"; + +milvus + { + + "store-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vector-store-pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vector-store-pinecone.jsonnet new file mode 100644 index 00000000..0f8d6a33 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vector-store-pinecone.jsonnet @@ -0,0 +1,161 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; + +{ + + "pinecone-cloud":: "aws", + "pinecone-region":: "us-east-1", + + "store-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vector-store-qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vector-store-qdrant.jsonnet new file mode 100644 index 00000000..0582cd2f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vector-store-qdrant.jsonnet @@ -0,0 +1,147 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local qdrant = import "stores/qdrant.jsonnet"; + +qdrant + { + + "store-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vertexai.jsonnet new file mode 100644 index 00000000..05f608ce --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vertexai.jsonnet @@ -0,0 +1,121 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vertexai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vertexai-" + key]:: value, + }, + + "vertexai-private-key":: "/vertexai/private.json", + "vertexai-region":: "us-central1", + "vertexai-max-output-tokens":: 4096, + "vertexai-temperature":: 0.0, + "vertexai-models":: models, + + "llm-models" +:: $["vertexai-models"], + + "text-completion" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion-rag") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm-service-gaudi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm-service-gaudi.jsonnet new file mode 100644 index 00000000..25c1d921 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm-service-gaudi.jsonnet @@ -0,0 +1,66 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "64.0", + "vllm-service-memory":: "64G", + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage").with_size("50G"); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-gaudi"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--tensor-parallel-size=8", + "--port", + "8899", + ]) + .with_runtime("habana") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + HUGGING_FACE_HUB_TOKEN: $["hf-token"], + HABANA_VISIBLE_DEVICES: "all", + VLLM_CACHE_ROOT: "/data", + }) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(8899, 8899, "vllm") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm-service-intel-gpu.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm-service-intel-gpu.jsonnet new file mode 100644 index 00000000..cdd98abc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm-service-intel-gpu.jsonnet @@ -0,0 +1,77 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "8.0", + "vllm-service-memory":: "16G", + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage").with_size("20G"); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-xpu"]) + .with_command([ + "python", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--dtype=float16", + "--device=xpu", + "--enforce-eager", + "--port", + "8899", + "--block-size", + "64", + "--gpu-memory-util", + "0.85", + "--trust-remote-code", + "--disable-sliding-window", + ]) + .with_environment({ + HF_TOKEN: $["hf-token"], + VLLM_USE_V1: "1", + W_LONG_MAX_MODEL_LEN: "1", + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + }) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(8899, 8899, "vllm") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm-service-nvidia.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm-service-nvidia.jsonnet new file mode 100644 index 00000000..42f23530 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm-service-nvidia.jsonnet @@ -0,0 +1,64 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-7B-Instruct-v0.3", + "vllm-service-cpus":: "0.5", + "vllm-service-memory":: "1G", + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage").with_size("50G"); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-nvidia"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--port", + "8899", + ]) + .with_runtime("nvidia") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + HUGGING_FACE_HUB_TOKEN: $["hf-token"], + VLLM_CACHE_ROOT: "/data", + }) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(8899, 8899, "vllm") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm.jsonnet new file mode 100644 index 00000000..63dfe84b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/vllm.jsonnet @@ -0,0 +1,110 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vllm.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-" + key]:: value, + }, + + "vllm-models":: models, + + "llm-models" +:: $["vllm-models"], + + "vllm-max-output-tokens":: 1024, + "vllm-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--concurrency", + std.toString($["text-completion-concurrency"]), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString($["text-completion-rag-concurrency"]), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/workbench-ui.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/workbench-ui.jsonnet new file mode 100644 index 00000000..f2048e47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/components/workbench-ui.jsonnet @@ -0,0 +1,32 @@ +local images = import "values/images.jsonnet"; + +{ + + "workbench-ui" +: { + + create:: function(engine) + + local container = + engine.container("workbench-ui") + .with_image(images["workbench-ui"]) + .with_limits("0.1", "256M") + .with_reservations("0.1", "256M") + .with_port(8888, 8888, "ui"); + + local containerSet = engine.containers( + "workbench-ui", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8888, 8888, "ui"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-aks-k8s.jsonnet new file mode 100644 index 00000000..c603a0d8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-aks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "engine/aks-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-docker-compose.jsonnet new file mode 100644 index 00000000..442d2cb7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-docker-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "engine/docker-compose.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-eks-k8s.jsonnet new file mode 100644 index 00000000..af8f8ed8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-eks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "engine/eks-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-gcp-k8s.jsonnet new file mode 100644 index 00000000..3d089a24 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-gcp-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "engine/gcp-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-minikube-k8s.jsonnet new file mode 100644 index 00000000..073358cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-minikube-k8s.jsonnet @@ -0,0 +1,26 @@ + +local engine = import "engine/minikube-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +// Extract resources using the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-noop.jsonnet new file mode 100644 index 00000000..cae548f7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-noop.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "engine/noop.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-ovh-k8s.jsonnet new file mode 100644 index 00000000..9c18f449 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-ovh-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "engine/ovh-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-podman-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-podman-compose.jsonnet new file mode 100644 index 00000000..442d2cb7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-podman-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "engine/docker-compose.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-scw-k8s.jsonnet new file mode 100644 index 00000000..98b79254 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-scw-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "engine/scw-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-tg-configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-tg-configuration.jsonnet new file mode 100644 index 00000000..4d4c6711 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config-to-tg-configuration.jsonnet @@ -0,0 +1,15 @@ + +local engine = import "engine/noop.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract configuration directly from patterns +patterns.configuration.configuration + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/config-composer.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/config-composer.jsonnet new file mode 100644 index 00000000..0e8db5b0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/config-composer.jsonnet @@ -0,0 +1,93 @@ +// Configuration Composer Module +// Orchestrates the complete configuration building process +// Combines all components into the final TrustGraph configuration + +local flow_builder = import "flow-builder.jsonnet"; +local interface_builder = import "interface-builder.jsonnet"; + +{ + // Main function to build the complete configuration + build: function(config_spec) + // Extract configuration parameters + local flow_classes = config_spec.flow_classes; + local default_flow_class = config_spec.default_flow_class; + local default_flow_id = config_spec.default_flow_id; + local flow_init_parameters = config_spec.flow_init_parameters; + + // Build all processors for the default flow + local class_processors = flow_builder.build_class_processors( + flow_classes, + default_flow_class, + flow_init_parameters + ); + + local flow_processors = flow_builder.build_flow_processors( + flow_classes, + default_flow_class, + default_flow_id, + flow_init_parameters + ); + + // Combine processors into flow objects + local processor_array = class_processors + flow_processors; + local flow_objects = flow_builder.build_flow_objects(processor_array); + local flows_active = flow_builder.merge_flow_objects(flow_objects); + + // Build interfaces for the default flow + local default_flow_interfaces = interface_builder.build_interfaces( + flow_classes, + default_flow_class, + default_flow_id, + flow_init_parameters + ); + + // Return object with nested configuration (for backwards compatibility) + { + // Create function (for backwards compatibility) + create: function(engine) {}, + + // The actual configuration object + configuration: { + // Prompts configuration + prompt: { + "system": config_spec.prompts["system-template"], + "template-index": std.objectFieldsAll(config_spec.prompts.templates), + } + { + ["template." + template.key]: template.value + for template in std.objectKeysValuesAll(config_spec.prompts.templates) + }, + + // Tools configuration + tool: { + [tool.id]: tool + for tool in config_spec.tools + }, + + // MCP configuration + mcp: config_spec.mcp, + + // Flow classes reference + "flow-classes": flow_classes, + + // Interface descriptions + "interface-descriptions": config_spec.interface_descriptions, + + // Flow instances + "flows": { + [default_flow_id]: { + "description": "Default processing flow", + "class-name": default_flow_class, + "interfaces": default_flow_interfaces, + "parameters": flow_init_parameters, + }, + }, + + // Active flow processors + "flows-active": flows_active, + + // Token costs and parameter types + "token-costs": config_spec.token_costs, + "parameter-types": config_spec.parameter_types, + }, + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/flow-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/flow-builder.jsonnet new file mode 100644 index 00000000..eff7c893 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/flow-builder.jsonnet @@ -0,0 +1,72 @@ +// Flow Builder Module +// Processes flow classes and builds complete flow configurations +// Handles {class}, {id}, and parameter substitutions + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds class-level processors with parameter substitution + // Processes the 'class' section of flow classes + build_class_processors: function(flow_classes, class_name, parameters) + [ + [ + // Replace {class} in the processor key + local key = std.strReplace(processor.key, "{class}", class_name); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // First replace {class}, then substitute parameters + local class_replaced = std.strReplace(field.value, "{class}", class_name); + param_processor.substitute_parameters(class_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_classes[class_name].class) + ], + + // Builds flow-level processors with parameter substitution + // Processes the 'flow' section of flow classes + build_flow_processors: function(flow_classes, class_name, flow_id, parameters) + [ + [ + // Replace both {class} and {id} in the processor key + local key = std.strReplace( + std.strReplace(processor.key, "{class}", class_name), + "{id}", flow_id + ); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // Replace {class} and {id}, then substitute parameters + local class_replaced = std.strReplace(field.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_classes[class_name].flow) + ], + + // Combines class and flow processors into flow objects + build_flow_objects: function(processor_array) + std.map( + function(item) { + [item[0][0]] +: { + [item[0][1]]: item[1] + } + }, + processor_array + ), + + // Merges all flow objects into a single flows_active configuration + merge_flow_objects: function(flow_objects) + std.foldr( + function(a, b) a + b, + flow_objects, + {} + ), +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/interface-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/interface-builder.jsonnet new file mode 100644 index 00000000..2143e600 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/interface-builder.jsonnet @@ -0,0 +1,30 @@ +// Interface Builder Module +// Processes flow class interfaces with parameter substitution +// Handles both string interfaces and nested object interfaces + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds interfaces for a specific flow class and instance + // Processes the 'interfaces' section of flow classes + build_interfaces: function(flow_classes, class_name, flow_id, parameters) + local interface_spec = flow_classes[class_name].interfaces; + { + [interface.key]: + if std.isString(interface.value) then + // Simple string interface - apply all substitutions + local class_replaced = std.strReplace(interface.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + else + // Complex object interface - process nested fields + { + [field.key]: + local class_replaced = std.strReplace(field.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(interface.value) + } + for interface in std.objectKeysValuesAll(interface_spec) + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/interface-descriptions.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/interface-descriptions.jsonnet new file mode 100644 index 00000000..f9dda8b0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/interface-descriptions.jsonnet @@ -0,0 +1,89 @@ +// Interface Descriptions Module +// Defines all external interfaces available in TrustGraph flows +// These are the 'endpoints' that external systems can interact with + +{ + // Document loading interfaces - for data ingestion + "document-load": { + "description": "Document loader", + "kind": "send", + "visible": true, + }, + "text-load": { + "description": "Text document loader", + "kind": "send", + "visible": true, + }, + + // Data storage interfaces - for processed data streams + "entity-contexts-load": { + "description": "Entity contexts loader", + "kind": "send", + }, + "triples-store": { + "description": "Triples loader", + "kind": "send", + }, + "graph-embeddings-store": { + "description": "Graph embeddings loader", + "kind": "send", + }, + "document-embeddings-store": { + "description": "Document embeddings loader", + "kind": "send", + }, + "objects-store": { + "description": "Object store", + "kind": "request-response", + }, + + // Query interfaces - for retrieving information + "graph-rag": { + "description": "GraphRAG service", + "kind": "request-response", + }, + "document-rag": { + "description": "ChunkRAG service", + "kind": "request-response", + }, + "triples": { + "description": "Triples query service", + "kind": "request-response", + }, + "graph-embeddings": { + "description": "Graph embeddings service", + "kind": "request-response", + }, + "document-embeddings": { + "description": "Document embeddings service", + "kind": "request-response", + }, + "objects": { + "description": "Object query service", + "kind": "request-response", + }, + + // Processing services - for text and data processing + "prompt": { + "description": "Prompt service", + "kind": "request-response", + }, + "agent": { + "description": "Agent service", + "kind": "request-response", + }, + "text-completion": { + "description": "Text completion service", + "kind": "request-response", + }, + + // Query translation services - for natural language queries + "nlp-query": { + "description": "NLP question to GraphQL service", + "kind": "request-response", + }, + "structured-query": { + "description": "Structured query service", + "kind": "request-response", + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/parameter-processor.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/parameter-processor.jsonnet new file mode 100644 index 00000000..f317ae9f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/parameter-processor.jsonnet @@ -0,0 +1,38 @@ +// Parameter Processing Module +// Handles dynamic parameter replacement in configuration values +// Replaces {parameter_name} placeholders with actual parameter values + +{ + // Applies parameter substitutions to string values + // Only processes strings - leaves other types unchanged + substitute_parameters: function(value, parameters) + if std.isString(value) then + std.foldl( + function(acc, param) + // Only do string replacement if param.value is a string + if std.isString(param.value) then + std.strReplace(acc, "{" + param.key + "}", param.value) + else + acc, // Skip replacement for non-string parameter values + std.objectKeysValuesAll(parameters), + value + ) + else + value, + + // Applies parameter substitutions to all values in an object + // Recursively processes nested objects and arrays + substitute_parameters_in_object: function(obj, parameters) + if std.isObject(obj) then + { + [key]: $.substitute_parameters_in_object(obj[key], parameters) + for key in std.objectFields(obj) + } + else if std.isArray(obj) then + [ + $.substitute_parameters_in_object(item, parameters) + for item in obj + ] + else + $.substitute_parameters(obj, parameters), +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/tools.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/tools.jsonnet new file mode 100644 index 00000000..a84cd0bb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/config/tools.jsonnet @@ -0,0 +1,52 @@ +// Tools Configuration Module +// Defines all available tools that can be used by agents and flows +// Each tool specifies its interface, arguments, and behavior + +[ + // Knowledge extraction tool - extracts structured knowledge from text + { + id: "knowledge-extraction", + name: "Knowledge extraction", + description: "Takes a chunk of text and extracts knowledge in definition and relationship formats. The input is a text chunk", + type: "prompt", + template: "agent-kg-extract", + arguments: [ + { + "name": "text", + "type": "string", + "description": "The text chunk", + } + ], + }, + + // Knowledge query tool - queries the knowledge base + { + id: "knowledge-query", + name: "Knowledge query", + description: "This tool queries a knowledge base that holds information about domain-specific information. The question should be a natural language question.", + type: "knowledge-query", + collection: "default", + arguments: [ + { + name: "question", + type: "string", + description: "A simple natural language question.", + } + ] + }, + + // LLM completion tool - general purpose text completion + { + id: "llm-completion", + name: "LLM text completion", + type: "text-completion", + description: "This tool queries an LLM for non-domain-specific information. The question should be a natural language question.", + arguments: [ + { + name: "question", + type: "string", + description: "The question which should be asked of the LLM.", + } + ] + } +] \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/aks-k8s.jsonnet new file mode 100644 index 00000000..07e050f2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/aks-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "disk.csi.azure.com", + parameters: { + // Standard disks (spinning magnetic), Locally Redundant Storage + // Cheapest, basically + skuName: "Standard_LRS", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/docker-compose.jsonnet new file mode 100644 index 00000000..954cde15 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/docker-compose.jsonnet @@ -0,0 +1,230 @@ +{ + + // Extract resources using the engine + package:: function(patterns) + std.foldl( + function(state, p) state + p.create(self), + std.objectValues(patterns), + {} + ), + + container:: function(name) + { + + local container = self, + + name:: name, + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_command:: function(x) self + { command: x }, + + with_runtime:: function(x) self + { runtime: x }, + + with_privileged:: function(x) self + { privileged: x }, + + with_ipc:: function(x) self + { ipc: x }, + + with_capability:: function(x) self + + if std.objectHas(container, "capability") then + { cap_add: container.capability + x } + else + { cap_add: [x], }, + + with_environment:: function(x) self + + if std.objectHas(container, "environment") then + { environment: container.environment + x } + else + { environment: x, }, + + with_device:: function(hdev, cdev) self + + if std.objectHas(container, "devices") then + { devices: container.devices + "%s:%s" % [hdev, cdev] } + else + { devices: [ "%s:%s" % [hdev, cdev] ], }, + + with_limits:: function(c, m) self + { + deploy +: { resources +: { + limits: { cpus: c, memory: m } + } }, + }, + + with_reservations:: function(c, m) self + { + deploy +: { resources +: { + reservations: { cpus: c, memory: m } + } }, + }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [vol.volid, mnt] + ] + else + [ + "%s:%s" % [vol.volid, mnt] + ] + }, + + with_port:: + function(src, dest, name) + self + { + ports: + if std.objectHas(container, "ports") then + container.ports + [ "%d:%d" % [src, dest] ] + else + [ "%d:%d" % [src, dest] ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj.with_environment( + { [x]: "${" + x + "}" } + ), + vars.variables, + self + ), + + restart: "on-failure:100", + + add:: function() { + services +: { + [container.name]: container, + } + } + + }, + + internalService:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + with_size:: function(size) self + { size: size }, + + add:: function() { + volumes +: { + [volume.name]: {} + } + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: "./" + dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + variables:: [], + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + }, + + add:: function() { + } + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.foldl( + function(state, c) state + c.add(), + cont.containers, + {} + ), + + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/eks-k8s.jsonnet new file mode 100644 index 00000000..3fc3f035 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/eks-k8s.jsonnet @@ -0,0 +1,46 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "ebs.csi.aws.com", + parameters: { + type: "gp3", + encrypted: "true", + iops: "6000", + throughput: "400", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/gcp-k8s.jsonnet new file mode 100644 index 00000000..71792426 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/gcp-k8s.jsonnet @@ -0,0 +1,44 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "pd.csi.storage.gke.io", + parameters: { + type: "pd-balanced", + "csi.storage.k8s.io/fstype": "ext4", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/k8s.jsonnet new file mode 100644 index 00000000..686f2029 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/k8s.jsonnet @@ -0,0 +1,358 @@ +{ + + container:: function(name) + { + + local container = self, + + name: name, + limits: {}, + reservations: {}, + ports: [], + volumes: [], + environment: [], + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_command:: function(x) self + { command: x }, + + with_environment:: function(x) self + { + environment: super.environment + [ + { + name: v.key, value: v.value + } + for v in std.objectKeysValues(x) + ], + }, + + with_limits:: function(c, m) self + { limits: { cpu: c, memory: m } }, + + with_reservations:: + function(c, m) self + { reservations: { cpu: c, memory: m } }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: super.volumes + [{ + volume: vol, mount: mnt + }] + }, + + with_port:: + function(src, dest, name) self + { + ports: super.ports + [ + { src: src, dest: dest, name : name } + ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj + { + environment: super.environment + [{ + name: x, + valueFrom: { + secretKeyRef: { + name: vars.name, + key: vars.keyMap[x], + } + } + }] + }, + vars.variables, + self + ), + + add:: function() [ + + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: container.name, + namespace: "trustgraph", + labels: { + app: container.name + } + }, + spec: { + replicas: 1, + selector: { + matchLabels: { + app: container.name, + } + }, + template: { + metadata: { + labels: { + app: container.name, + } + }, + spec: { + containers: [ + { + name: container.name, + image: container.image, + + // FIXME: Make everything run as + // root. Needed to get filesystems + // to be accessible. There's a + // better way of doing this? + securityContext: { + runAsUser: 0, + runAsGroup: 0, + }, + + resources: { + requests: container.reservations, + limits: container.limits + }, + } + ( + if std.length(container.ports) > 0 then + { + ports: [ + { + hostPort: port.src, + containerPort: port.dest, + } + for port in container.ports + ] + } else + {}) + + + (if std.objectHas(container, "command") then + { command: container.command } + else {}) + + + (if std.length(container.environment) > 0 then + { + env: container.environment, + } + else {}) + + + (if std.length(container.volumes) > 0 then + { + volumeMounts: [ + { + mountPath: vol.mount, + name: vol.volume.name, + } + for vol in container.volumes + ] + } + + else + {} + ) + ], + volumes: [ + vol.volume.volRef() + for vol in container.volumes + + ] + } + }, + } + {} + + } + + ] + + }, + + // Just an alias + internalService:: self.service, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + ports: [], + + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + + add:: function() [ + + { + + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + storageClassName: "tg", + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + data: parts + }, + ], + + + volRef:: function() { + name: volume.name, + configMap: { name: volume.name }, + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "Secret", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + data: { + [item.key]: std.base64(item.value) + for item in std.objectKeysValues(parts) + } + }, + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + variables: [], + keyMap: {}, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + }, + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + keyMap: super.keyMap + { [name]: key }, + }, + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.flattenArrays( + [ c.add() for c in cont.containers ] + ), + + }, + + resources:: function(res) + + std.flattenArrays( + [ c.add() for c in res ] + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/minikube-k8s.jsonnet new file mode 100644 index 00000000..858b17ad --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/minikube-k8s.jsonnet @@ -0,0 +1,115 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList, + + volume:: function(name) + { + local volume = self, + name: name, + with_size:: function(size) self + { size: size }, + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolume", + metadata: { + name: volume.name, + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + capacity: { + storage: volume.size, + }, + persistentVolumeReclaimPolicy: "Delete", + hostPath: { + path: "/data/pv-" + volume.name, + }, + } + }, + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + service:: function(containers) + { + local service = self, + name: containers.name, + ports: [], + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + add:: function() [ + { + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + type: "LoadBalancer", + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/noop.jsonnet new file mode 100644 index 00000000..1f384648 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/noop.jsonnet @@ -0,0 +1,79 @@ +{ + + // Extract resources usnig the engine + package:: function(patterns) {}, + + container:: function(name) { + + with_image:: function(x) self + {}, + + with_user:: function(x) self + {}, + + with_command:: function(x) self + {}, + + with_runtime:: function(x) self + {}, + + with_privileged:: function(x) self + {}, + + with_ipc:: function(x) self + {}, + + with_capability:: function(x) self + {}, + + with_environment:: function(x) self + {}, + + with_device:: function(hdev, cdev) self + {}, + + with_limits:: function(c, m) self + {}, + + with_reservations:: function(c, m) self + {}, + + with_volume_mount:: self + {}, + + with_port:: function(src, dest, name) self + {}, + + with_env_var_secrets:: function(vars) self + {}, + + add:: function() {}, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + service:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + volume:: function(name) { + with_size:: function(size) self + {}, + add:: function() {}, + }, + + configVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + secretVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self + {}, + add:: function() {}, + }, + + containers:: function(name, containers) { + add:: function() {}, + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/ovh-k8s.jsonnet new file mode 100644 index 00000000..15d57c83 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/ovh-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "cinder.csi.openstack.org", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", + parameters: { + availability: "nova", + fsType: "ext4", + type: "high-speed", + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/scw-k8s.jsonnet new file mode 100644 index 00000000..eed23968 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/engine/scw-k8s.jsonnet @@ -0,0 +1,40 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "csi.scaleway.com", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/agent-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/agent-extract.jsonnet new file mode 100644 index 00000000..143e736a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/agent-extract.jsonnet @@ -0,0 +1,35 @@ +// Agent-based extraction module +// Uses AI agents for more sophisticated knowledge extraction from text +// Leverages agent tools and reasoning for complex extraction tasks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - internal agent extraction service + "interfaces" +: { + }, + + // No configurable parameters for agent extraction + "parameters" +: { + }, + + // Flow-level processors for agent-based extraction + "flow" +: { + // Agent-based knowledge extraction processor + // Uses AI agents with tools to extract structured knowledge + "kg-extract-agent:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output knowledge triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "agent-request": request("agent:{id}"), // Agent service requests + "agent-response": response("agent:{id}"), // Agent service responses + }, + }, + + // No class-level processors needed + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/agent.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/agent.jsonnet new file mode 100644 index 00000000..19afe660 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/agent.jsonnet @@ -0,0 +1,59 @@ +// Agent management module +// Provides AI agent orchestration and tool integration +// Manages agent conversations, tool calls, and response coordination +// Supports MCP tools, GraphRAG, and structured queries + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // External interfaces for agent operations + "interfaces" +: { + "agent": request_response("agent:{id}"), // Main agent service interface + "mcp-tool": request_response("mcp-tool:{id}"), // MCP tool execution interface + }, + + // No configurable parameters for agent management + "parameters" +: { + }, + + // Flow-level processors for agent management + "flow" +: { + // Agent manager orchestrates agent conversations and tool usage + "agent-manager:{id}": { + // Agent communication channels + request: request("agent:{id}"), // Incoming agent requests + next: request("agent:{id}"), // Multi-turn conversation support + response: response("agent:{id}"), // Agent responses + + // LLM and prompt services + "text-completion-request": request("text-completion:{id}"), // LLM requests + "text-completion-response": response("text-completion:{id}"), // LLM responses + "prompt-request": request("prompt:{id}"), // Prompt processing + "prompt-response": response("prompt:{id}"), + + // Tool integrations + "mcp-tool-request": request("mcp-tool:{id}"), // MCP tool calls + "mcp-tool-response": response("mcp-tool:{id}"), + "graph-rag-request": request("graph-rag:{id}"), // GraphRAG queries + "graph-rag-response": response("graph-rag:{id}"), + "structured-query-request": request("structured-query:{id}"), // Structured data queries + "structured-query-response": response("structured-query:{id}"), + }, + + // MCP tool executor for agent tool usage + "mcp-tool:{id}": { + request: request("mcp-tool:{id}"), // Tool invocation requests + response: response("mcp-tool:{id}"), // Tool execution results + "text-completion-request": request("text-completion:{id}"), // LLM for tool reasoning + "text-completion-response": response("text-completion:{id}"), + }, + + }, + + // Class-level processors for agent-related services + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/documentrag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/documentrag.jsonnet new file mode 100644 index 00000000..65910684 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/documentrag.jsonnet @@ -0,0 +1,100 @@ +// Document RAG (Retrieval Augmented Generation) module +// Implements document-based RAG using chunk embeddings +// Provides semantic search and context-aware question answering +// Supports MCP (Model Context Protocol) tool integration + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces for document RAG functionality + "interfaces" +: { + // Document embedding storage and retrieval + "document-embeddings-store": flow("document-embeddings-store:{id}"), // Embedding storage stream + "document-rag": request_response("document-rag:{id}"), // Main document RAG interface + "document-embeddings": request_response("document-embeddings:{id}"), // Document embedding queries + + // Supporting services + "embeddings": request_response("embeddings:{id}"), // General embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing + "mcp-tool": request_response("mcp-tool:{id}"), // MCP tool integration + "text-completion": request_response("text-completion:{id}"), // LLM text completion + }, + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors for document embedding and storage + "flow" +: { + "document-embeddings:{id}": { + input: flow("chunk-load:{id}"), + output: flow("document-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "de-write:{id}": { + input: flow("document-embeddings-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "document-rag:{id}": { + request: request("document-rag:{id}"), + response: response("document-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "document-embeddings-request": request("document-embeddings:{id}"), + "document-embeddings-response": response("document-embeddings:{id}"), + }, + "de-query:{id}": { + request: request("document-embeddings:{id}"), + response: response("document-embeddings:{id}"), + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "mcp-tool:{id}": { + request: request("mcp-tool:{id}"), + response: response("mcp-tool:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + // Class-level processors for document RAG operations + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/flow-classes.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/flow-classes.jsonnet new file mode 100644 index 00000000..1a1adabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/flow-classes.jsonnet @@ -0,0 +1,105 @@ +// TrustGraph Flow Classes Configuration +// Defines different flow combinations for various use cases +// Each flow class combines multiple functional modules to create complete processing pipelines +// +// Available modules: +// - graphrag: Graph-based RAG with knowledge graphs +// - documentrag: Document-based RAG with chunk embeddings +// - structured: Structured data processing and NLP queries +// - agent: AI agent orchestration and tool integration +// - load: Document loading and preprocessing +// - kg-base: Basic knowledge extraction from text +// - agent-extract: Agent-based knowledge extraction +// - kgcore: Knowledge graph core storage + +// Import all the modular flow components +local graphrag_part = import "graphrag.jsonnet"; +local kg_base_part = import "kg-base.jsonnet"; +local onto_base_part = import "onto-base.jsonnet"; +local agent_extract_part = import "agent-extract.jsonnet"; +local structured_part = import "structured.jsonnet"; +local documentrag_part = import "documentrag.jsonnet"; +local agent_part = import "agent.jsonnet"; +local load_part = import "load.jsonnet"; +local kgcore_part = import "kgcore.jsonnet"; + +{ + + // Complete TrustGraph system with all capabilities + // Includes GraphRAG, DocumentRAG, structured data processing, and knowledge cores + "everything": { + description: "GraphRAG, DocumentRAG, structured data + knowledge cores", + tags: [ + "document-rag", "graph-rag", "knowledge-extraction", + "structured-data", "kgcore" + ], + } + + graphrag_part + documentrag_part + agent_part + load_part + + kg_base_part + structured_part, + + // Dual RAG system without knowledge core creation + // Combines both document and graph-based retrieval + "document-rag+graph-rag": { + description: "Supports GraphRAG and document RAG, no core creation", + tags: ["document-rag", "graph-rag", "knowledge-extraction"], + } + + graphrag_part + documentrag_part + agent_part + load_part + kg_base_part, + + // Graph-based RAG only + // Uses knowledge graphs for context-aware question answering + "graph-rag": { + description: "GraphRAG only", + tags: ["graph-rag", "knowledge-extraction"], + } + + graphrag_part + agent_part + load_part + kg_base_part, + + // Graph-based RAG only + // Uses knowledge graphs for context-aware question answering + "onto-rag": { + description: "Ontology RAG only", + tags: ["graph-rag", "knowledge-extraction"], + } + + graphrag_part + agent_part + load_part + onto_base_part, + + // Document-based RAG only + // Uses document embeddings for semantic search and answers + "document-rag": { + description: "DocumentRAG only", + tags: ["document-rag"], + } + + documentrag_part + load_part, + + // Full RAG system with knowledge core creation + // Includes both RAG types plus persistent knowledge storage + "document-rag+graph-rag+kgcore": { + description: "GraphRAG + DocumentRAG + knowledge core creation", + tags: ["document-rag", "graph-rag", "knowledge-extraction"], + } + + graphrag_part + documentrag_part + agent_part + load_part + + kgcore_part + kg_base_part, + + // GraphRAG with advanced agent-based extraction + // Uses AI agents for sophisticated knowledge extraction + "graph-rag+agent-extract": { + description: "GraphRAG + agent extract", + tags: ["graph-rag", "knowledge-extraction", "agent-extract"], + } + + graphrag_part + agent_part + load_part + agent_extract_part, + + // GraphRAG with structured data processing + // Combines knowledge graphs with structured data queries + "graph-rag+structured-data": { + description: "GraphRAG + structured data", + tags: ["graph-rag", "knowledge-extraction", "structured-data"], + } + + graphrag_part + agent_part + load_part + structured_part, + + // Structured data processing only + // Handles structured data extraction and NLP queries + "structured-data": { + description: "Structured data only", + tags: ["knowledge-extraction", "structured-data"], + } + + agent_part + load_part + structured_part, + +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/graphrag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/graphrag.jsonnet new file mode 100644 index 00000000..94cf5750 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/graphrag.jsonnet @@ -0,0 +1,107 @@ +// GraphRAG flow configuration module +// Implements graph-based retrieval augmented generation (GraphRAG) functionality +// Handles knowledge graph storage, embeddings, and graph-based question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces exposed by the GraphRAG flow + "interfaces" +: { + // Data ingestion interfaces for graph construction + "entity-contexts-load": flow("entity-contexts-load:{id}"), // Entity context data stream + "triples-store": flow("triples-store:{id}"), // RDF triples storage stream + "graph-embeddings-store": flow("graph-embeddings-store:{id}"), // Graph embedding storage + + // Query interfaces for graph-based operations + "graph-rag": request_response("graph-rag:{id}"), // Main GraphRAG query interface + "triples": request_response("triples:{id}"), // Triple store queries + "graph-embeddings": request_response("graph-embeddings:{id}"), // Graph embedding queries + + // Supporting services + "embeddings": request_response("embeddings:{id}"), // General embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing service + "text-completion": request_response("text-completion:{id}"), // LLM text completion + }, + + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors - handle data streams for a specific flow instance + "flow" +: { + "graph-embeddings:{id}": { + input: flow("entity-contexts-load:{id}"), + output: flow("graph-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "triples-write:{id}": { + input: flow("triples-store:{id}"), + }, + "ge-write:{id}": { + input: flow("graph-embeddings-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "graph-rag:{id}": { + request: request("graph-rag:{id}"), + response: response("graph-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "graph-embeddings-request": request("graph-embeddings:{id}"), + "graph-embeddings-response": response("graph-embeddings:{id}"), + "triples-request": request("triples:{id}"), + "triples-response": response("triples:{id}"), + }, + "triples-query:{id}": { + request: request("triples:{id}"), + response: response("triples:{id}"), + }, + "ge-query:{id}": { + request: request("graph-embeddings:{id}"), + response: response("graph-embeddings:{id}"), + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + // Class-level processors - shared across all flow instances of this class + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/helpers.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/helpers.jsonnet new file mode 100644 index 00000000..eabb3bf4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/helpers.jsonnet @@ -0,0 +1,29 @@ +// Helper functions for flow configuration +// Provides utility functions for constructing flow, request, and response URIs +// used throughout the TrustGraph flow configuration system + +// Creates a persistent flow URI for data streams +// Persistent flows retain messages until consumed +local flow(x) = "persistent://tg/flow/" + x; + +// Creates a non-persistent request URI for request-response patterns +// Non-persistent means messages are not retained if no consumer is present +local request(x) = "non-persistent://tg/request/" + x; + +// Creates a non-persistent response URI for request-response patterns +local response(x) = "non-persistent://tg/response/" + x; + +// Creates a request-response pair for bidirectional communication +// Returns an object with both request and response URIs +local request_response(x) = { + request: request(x), + response: response(x), +}; + +// Export all helper functions for use in other modules +{ + flow: flow, + request: request, + response: response, + request_response: request_response, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/kg-base.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/kg-base.jsonnet new file mode 100644 index 00000000..215dd791 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/kg-base.jsonnet @@ -0,0 +1,44 @@ +// Knowledge Graph Base extraction module +// Provides basic knowledge extraction capabilities from text chunks +// Extracts entity definitions and relationships using prompt-based processing + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts entity definitions from text chunks + // Identifies and defines key entities mentioned in the text + "kg-extract-definitions:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output definition triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition extraction prompts + "prompt-response": response("prompt:{id}"), + }, + + // Extracts relationships between entities + // Identifies how entities are connected and interact + "kg-extract-relationships:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output relationship triples + "prompt-request": request("prompt:{id}"), // Relationship extraction prompts + "prompt-response": response("prompt:{id}"), + }, + }, + + // No class-level processors needed + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/kgcore.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/kgcore.jsonnet new file mode 100644 index 00000000..4a857e66 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/kgcore.jsonnet @@ -0,0 +1,31 @@ +// Knowledge Graph Core storage module +// Handles persistent storage of knowledge graph data +// Consolidates triples and graph embeddings into permanent storage +// Creates the core knowledge base for long-term use + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; + +{ + // No external interfaces - internal storage service + "interfaces" +: { + }, + + // No configurable parameters for core storage + "parameters" +: { + }, + + // Flow-level processors for knowledge graph storage + "flow" +: { + // Knowledge graph store consolidates extracted knowledge + // Takes processed triples and embeddings and stores them permanently + "kg-store:{id}": { + "triples-input": flow("triples-store:{id}"), // Input RDF triples stream + "graph-embeddings-input": flow("graph-embeddings-store:{id}"), // Input graph embeddings + }, + }, + + // No class-level processors needed + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/llm-parameters.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/llm-parameters.jsonnet new file mode 100644 index 00000000..a99f72ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/llm-parameters.jsonnet @@ -0,0 +1,59 @@ +{ + + // LLM model selection for normal LLM + "llm-model": { + "type": "llm-model", + "description": "LLM model", + "order": 1, + "advanced": false, + }, + + // LLM model for RAG operations + "llm-rag-model": { + "type": "llm-model", + "description": "LLM model for RAG", + "order": 2, + "advanced": true, + "controlled-by": "llm-model", + }, + + // LLM model selection for normal LLM + "llm-temperature": { + "type": "llm-temperature", + "description": "LLM temperature", + "order": 3, + "advanced": true, + }, + + // LLM model selection for normal LLM + "llm-rag-temperature": { + "type": "llm-temperature", + "description": "LLM temperature for RAG", + "order": 4, + "advanced": true, + }, + + "embeddings-model": { + "type": "embeddings-model", + "description": "Embeddings model", + "order": 5, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-size": { + "type": "chunk-size", + "description": "Chunk size", + "order": 6, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-overlap": { + "type": "chunk-overlap", + "description": "Chunk overlap", + "order": 7, + "advanced": true, + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/load.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/load.jsonnet new file mode 100644 index 00000000..eacf60dd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/load.jsonnet @@ -0,0 +1,50 @@ +// Document loading and preprocessing module +// Handles document ingestion, format conversion, and chunking +// Converts PDFs to text and splits documents into processable chunks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + + // External interfaces for document loading + "interfaces" +: { + "document-load": flow("document-load:{id}"), // Raw document input stream + "text-load": flow("text-document-load:{id}"), // Text document stream + "embeddings": request_response("embeddings:{id}"), // Embedding service for chunks + }, + + // No configurable parameters for document loading + "parameters" +: { + }, + + // Flow-level processors for document preprocessing + "flow" +: { + // PDF decoder converts PDF documents to text + "pdf-decoder:{id}": { + input: flow("document-load:{id}"), // Raw PDF input + output: flow("text-document-load:{id}"), // Extracted text output + }, + + // Chunker splits documents into smaller, processable pieces + "chunker:{id}": { + input: flow("text-document-load:{id}"), // Full text documents + output: flow("chunk-load:{id}"), // Document chunks for processing + "chunk-size": "{chunk-size}", // Chunk size + "chunk-overlap": "{chunk-overlap}", // Overlap between chunks + }, + // Embedding service for converting text chunks to vectors + "embeddings:{id}": { + request: request("embeddings:{id}"), // Embedding requests + response: response("embeddings:{id}"), // Embedding responses + model: "{embeddings-model}", + }, + }, + + // Class-level processors for document loading services + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/onto-base.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/onto-base.jsonnet new file mode 100644 index 00000000..2236f24d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/onto-base.jsonnet @@ -0,0 +1,39 @@ +// Knowledge Graph Base extraction module +// Provides basic knowledge extraction capabilities from text chunks +// Extracts entity definitions and relationships using prompt-based processing + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts using ontology definitions + "kg-extract-ontology:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition + // extraction prompts + "prompt-response": response("prompt:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + + }, + + // No class-level processors needed + "class" +: { + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/ontorag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/ontorag.jsonnet new file mode 100644 index 00000000..94cf5750 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/ontorag.jsonnet @@ -0,0 +1,107 @@ +// GraphRAG flow configuration module +// Implements graph-based retrieval augmented generation (GraphRAG) functionality +// Handles knowledge graph storage, embeddings, and graph-based question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces exposed by the GraphRAG flow + "interfaces" +: { + // Data ingestion interfaces for graph construction + "entity-contexts-load": flow("entity-contexts-load:{id}"), // Entity context data stream + "triples-store": flow("triples-store:{id}"), // RDF triples storage stream + "graph-embeddings-store": flow("graph-embeddings-store:{id}"), // Graph embedding storage + + // Query interfaces for graph-based operations + "graph-rag": request_response("graph-rag:{id}"), // Main GraphRAG query interface + "triples": request_response("triples:{id}"), // Triple store queries + "graph-embeddings": request_response("graph-embeddings:{id}"), // Graph embedding queries + + // Supporting services + "embeddings": request_response("embeddings:{id}"), // General embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing service + "text-completion": request_response("text-completion:{id}"), // LLM text completion + }, + + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors - handle data streams for a specific flow instance + "flow" +: { + "graph-embeddings:{id}": { + input: flow("entity-contexts-load:{id}"), + output: flow("graph-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "triples-write:{id}": { + input: flow("triples-store:{id}"), + }, + "ge-write:{id}": { + input: flow("graph-embeddings-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "graph-rag:{id}": { + request: request("graph-rag:{id}"), + response: response("graph-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "graph-embeddings-request": request("graph-embeddings:{id}"), + "graph-embeddings-response": response("graph-embeddings:{id}"), + "triples-request": request("triples:{id}"), + "triples-response": response("triples:{id}"), + }, + "triples-query:{id}": { + request: request("triples:{id}"), + response: response("triples:{id}"), + }, + "ge-query:{id}": { + request: request("graph-embeddings:{id}"), + response: response("graph-embeddings:{id}"), + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + // Class-level processors - shared across all flow instances of this class + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/structured.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/structured.jsonnet new file mode 100644 index 00000000..45bd35d9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/flows/structured.jsonnet @@ -0,0 +1,108 @@ +// Structured data processing module +// Handles extraction and querying of structured data objects +// Provides natural language to GraphQL query capabilities +// Supports structured data storage and retrieval + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces for structured data operations + "interfaces" +: { + // Supporting services + "embeddings": request_response("embeddings:{id}"), // Embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing + "text-completion": request_response("text-completion:{id}"), // LLM completion + + // Structured data storage and querying + "objects-store": flow("objects-store:{id}"), // Object storage stream + "objects": request_response("objects:{id}"), // Object query service + + // Query interfaces + "nlp-query": request_response("nlp-query:{id}"), // NLP to GraphQL translation + "structured-query": request_response("structured-query:{id}"), // Structured query execution + "structured-diag": request_response("structured-diag:{id}"), // Query diagnostics + }, + + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors for structured data extraction + "flow" +: { + "kg-extract-objects:{id}": { + input: flow("chunk-load:{id}"), + output: flow("objects-store:{id}"), + "entity-contexts": flow("entity-contexts-load:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + "objects-write:{id}": { + input: flow("objects-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "objects-query:{id}": { + request: request("objects:{id}"), + response: response("objects:{id}"), + }, + "nlp-query:{id}": { + request: request("nlp-query:{id}"), + response: response("nlp-query:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + }, + "structured-query:{id}": { + request: request("structured-query:{id}"), + response: response("structured-query:{id}"), + "nlp-query-request": request("nlp-query:{id}"), + "nlp-query-response": response("nlp-query:{id}"), + "objects-query-request": request("objects:{id}"), + "objects-query-response": response("objects:{id}"), + }, + "structured-diag:{id}": { + request: request("structured-diag:{id}"), + response: response("structured-diag:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + // Class-level processors for structured data operations + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/mcp/ddg-mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/mcp/ddg-mcp-server.jsonnet new file mode 100644 index 00000000..bb9157f4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/mcp/ddg-mcp-server.jsonnet @@ -0,0 +1,48 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "ddg-mcp-server-port":: 9870, + + "ddg-mcp-server" +: { + + create:: function(engine) + + local port = $["ddg-mcp-server-port"]; + + local container = + engine.container("ddg-mcp-server") + .with_image(images["ddg-mcp-server"]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "ddg-mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + mcp +:: { + "duckduckgo": { + "remote-name": "search", + local port = $["ddg-mcp-server-port"], + local url = "http://ddg-mcp-server:%s/mcp" % port, + "url": url, + } + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/azure-openai.jsonnet new file mode 100644 index 00000000..0e3ea907 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/azure-openai.jsonnet @@ -0,0 +1,9 @@ +// Azure OpenAI LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-4o", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/azure.jsonnet new file mode 100644 index 00000000..6ffa5fdf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/azure.jsonnet @@ -0,0 +1,9 @@ +// Azure LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/bedrock.jsonnet new file mode 100644 index 00000000..df930e59 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/bedrock.jsonnet @@ -0,0 +1,63 @@ +// AWS Bedrock LLM Model Definitions +// Defines available models and their configurations for AWS Bedrock + +{ + "type": "string", + "description": "LLM model to use", + "default": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + "enum": [ + { + id: "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + description: "Claude Sonnet 4.5 (smartest for complex agents and coding)" + }, + { + id: "global.anthropic.claude-opus-4-5-20251101-v1:0", + description: "Claude Opus 4.5 (maximum intelligence)" + }, + { + id: "global.anthropic.claude-haiku-4-5-20251001-v1:0", + description: "Claude Haiku 4.5 (fastest with near-frontier intelligence)" + }, + { + id: "global.anthropic.claude-opus-4-1-20250805-v1:0", + description: "Claude Opus 4.1 (specialized reasoning)" + }, + { + id: "global.anthropic.claude-sonnet-4-20250514-v1:0", + description: "Claude Sonnet 4.0" + }, + { + id: "global.anthropic.claude-opus-4-20250514-v1:0", + description: "Claude Opus 4.0" + }, + { + id: "anthropic.claude-3-5-haiku-20241022-v1:0", + description: "Claude 3.5 Haiku" + }, + { + id: "anthropic.claude-3-haiku-20240307-v1:0", + description: "Claude 3 Haiku" + }, + { + id: "meta.llama3-1-405b-instruct-v1:0", + description: "Llama 3.1 405B Instruct" + }, + { + id: "meta.llama3-1-70b-instruct-v1:0", + description: "Llama 3.1 70B Instruct" + }, + { + id: "meta.llama3-1-8b-instruct-v1:0", + description: "Llama 3.1 8B Instruct" + }, + { + id: "mistral.mistral-large-2407-v1:0", + description: "Mistral Large" + }, + { + id: "mistral.mixtral-8x7b-instruct-v0:1", + description: "Mixtral 8x7B Instruct" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/chunking-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/chunking-param-types.jsonnet new file mode 100644 index 00000000..f82fa3f7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/chunking-param-types.jsonnet @@ -0,0 +1,25 @@ +// Chunk parameter type definitions + +{ + "chunk-size": { + "type": "integer", + "description": "Chunk size", + "placeholder": 2000, + "helper": "An integer, usually 2000 .. 8000", + "default": 2000, + "min": 0, + "max": 32768, + "required": true + }, + "chunk-overlap": { + "type": "integer", + "description": "Chunk overlap", + "placeholder": 50, + "helper": "An integer, usually 50 .. 100", + "default": 50, + "min": 0, + "max": 8000, + "required": true + }, +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/claude.jsonnet new file mode 100644 index 00000000..40fd434b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/claude.jsonnet @@ -0,0 +1,35 @@ +// Claude LLM Model Definitions +// Defines available models and their configurations for Anthropic's Claude + +{ + "type": "string", + "description": "LLM model to use", + "default": "claude-sonnet-4-5-20250929", + "enum": [ + { + id: "claude-sonnet-4-5-20250929", + description: "Claude Sonnet 4.5 (complex agents + coding)" + }, + { + id: "claude-opus-4-5-20251101", + description: "Claude Opus 4.5 (maximum intelligence)" + }, + { + id: "claude-haiku-4-5-20251001", + description: "Claude Haiku 4.5 (fast)" + }, + { + id: "claude-opus-4-1-20250805", + description: "Claude Opus 4.1 (specialized reasoning)" + }, + { + id: "claude-sonnet-4-20250514", + description: "Claude Sonnet 4.0" + }, + { + id: "claude-3-5-haiku-20241022", + description: "Claude 3.5 Haiku" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/cohere.jsonnet new file mode 100644 index 00000000..5a5865a3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/cohere.jsonnet @@ -0,0 +1,35 @@ +// Cohere LLM Model Definitions +// Defines available models and their configurations for Cohere + +{ + "type": "string", + "description": "LLM model to use", + "default": "command-r-plus-08-2024", + "enum": [ + { + id: "command-r-plus-08-2024", + description: "Command R+ (August 2024)" + }, + { + id: "command-r-08-2024", + description: "Command R (August 2024)" + }, + { + id: "command-r-plus", + description: "Command R+ (legacy)" + }, + { + id: "command-r", + description: "Command R (legacy)" + }, + { + id: "command", + description: "Command" + }, + { + id: "command-light", + description: "Command Light" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..477e25dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/embeddings-fastembed.jsonnet @@ -0,0 +1,127 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "sentence-transformers/all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "BAAI/bge-small-en-v1.5", + "description": "bge-small-en-v1.5" + }, + { + "id": "BAAI/bge-small-zh-v1.5", + "description": "bge-small-zh-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-xs", + "description": "snowflake-arctic-embed-xs" + }, + { + "id": "jinaai/jina-embeddings-v2-small-en", + "description": "jina-embeddings-v2-small-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "snowflake/snowflake-arctic-embed-s", + "description": "snowflake-arctic-embed-s" + }, + { + "id": "BAAI/bge-small-en", + "description": "bge-small-en" + }, + { + "id": "BAAI/bge-base-en-v1.5", + "description": "bge-base-en-v1.5" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "description": "paraphrase-multilingual-MiniLM-L12-v2" + }, + { + "id": "Qdrant/clip-ViT-B-32-text", + "description": "clip-ViT-B-32-text" + }, + { + "id": "jinaai/jina-embeddings-v2-base-de", + "description": "jina-embeddings-v2-base-de" + }, + { + "id": "BAAI/bge-base-en", + "description": "bge-base-en" + }, + { + "id": "snowflake/snowflake-arctic-embed-m", + "description": "snowflake-arctic-embed-m" + }, + { + "id": "thenlper/gte-base", + "description": "gte-base" + }, + { + "id": "jinaai/jina-embeddings-v2-base-en", + "description": "jina-embeddings-v2-base-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1", + "description": "nomic-embed-text-v1" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5", + "description": "nomic-embed-text-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-m-long", + "description": "snowflake-arctic-embed-m-long" + }, + { + "id": "jinaai/jina-clip-v1", + "description": "jina-clip-v1" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + { + "id": "jinaai/jina-embeddings-v2-base-es", + "description": "jina-embeddings-v2-base-es" + }, + { + "id": "jinaai/jina-embeddings-v2-base-code", + "description": "jina-embeddings-v2-base-code" + }, + { + "id": "jinaai/jina-embeddings-v2-base-zh", + "description": "jina-embeddings-v2-base-zh" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "description": "paraphrase-multilingual-mpnet-base-v2" + }, + { + "id": "snowflake/snowflake-arctic-embed-l", + "description": "snowflake-arctic-embed-l" + }, + { + "id": "BAAI/bge-large-en-v1.5", + "description": "bge-large-en-v1.5" + }, + { + "id": "thenlper/gte-large", + "description": "gte-large" + }, + { + "id": "intfloat/multilingual-e5-large", + "description": "multilingual-e5-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/embeddings-huggingface.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/embeddings-huggingface.jsonnet new file mode 100644 index 00000000..32254a2b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/embeddings-huggingface.jsonnet @@ -0,0 +1,31 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "all-mpnet-base-v2", + "description": "all-mpnet-base-v2" + }, + { + "id": "all-distilroberta-v1", + "description": "all-distilroberta-v1" + }, + { + "id": "stsb-bert-large", + "description": "stsb-bert-large" + }, + { + "id": "sentence-camembert-large", + "description": "sentence-camembert-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/embeddings-ollama.jsonnet new file mode 100644 index 00000000..c3f29fbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/embeddings-ollama.jsonnet @@ -0,0 +1,23 @@ +// Embeddings model definitions for Ollama +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "all-minilm:latest", + "enum": [ + { + "id": "all-minilm:latest", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/googleaistudio.jsonnet new file mode 100644 index 00000000..a8891932 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/googleaistudio.jsonnet @@ -0,0 +1,67 @@ +// Google AI Studio LLM Model Definitions +// Defines available models and their configurations for Google AI Studio + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 2.5 models (latest generation) + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-exp", + description: "Gemini 2.0 Flash (experimental)" + }, + + + // Claude models on VertexAI + { + id: "claude-3-5-sonnet@20241022", + description: "Claude 3.5 Sonnet (via VertexAI)" + }, + { + id: "claude-3-5-haiku@20241022", + description: "Claude 3.5 Haiku (via VertexAI)" + }, + { + id: "claude-3-opus@20240229", + description: "Claude 3 Opus (via VertexAI)" + }, + { + id: "claude-3-sonnet@20240229", + description: "Claude 3 Sonnet (via VertexAI)" + }, + { + id: "claude-3-haiku@20240307", + description: "Claude 3 Haiku (via VertexAI)" + }, + + // Llama models on VertexAI + { + id: "llama3-405b-instruct-maas", + description: "Llama 3 405B Instruct (via VertexAI)" + }, + { + id: "llama3-70b-instruct-maas", + description: "Llama 3 70B Instruct (via VertexAI)" + }, + { + id: "llama3-8b-instruct-maas", + description: "Llama 3 8B Instruct (via VertexAI)" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/llamafile.jsonnet new file mode 100644 index 00000000..f6480aa9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/llamafile.jsonnet @@ -0,0 +1,9 @@ +// LlamaFile LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/lmstudio.jsonnet new file mode 100644 index 00000000..1c88da3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/lmstudio.jsonnet @@ -0,0 +1,150 @@ +// LMStudio LLM Model Definitions +// Defines available models and their configurations for LMStudio + +{ + "type": "string", + "description": "LLM model to use", + "default": "llama3.1:70b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/mistral.jsonnet new file mode 100644 index 00000000..23b7bf79 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/mistral.jsonnet @@ -0,0 +1,39 @@ +// Mistral LLM Model Definitions +// Defines available models and their configurations for Mistral AI + +{ + "type": "string", + "description": "LLM model to use", + "default": "mistral-large-latest", + "enum": [ + { + id: "mistral-large-latest", + description: "Mistral Large (latest)" + }, + { + id: "mistral-medium-latest", + description: "Mistral Medium (latest)" + }, + { + id: "mistral-small-latest", + description: "Mistral Small (latest)" + }, + { + id: "open-mistral-nemo", + description: "Open Mistral Nemo" + }, + { + id: "open-mistral-7b", + description: "Open Mistral 7B" + }, + { + id: "open-mixtral-8x7b", + description: "Open Mixtral 8x7B" + }, + { + id: "open-mixtral-8x22b", + description: "Open Mixtral 8x22B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/ollama.jsonnet new file mode 100644 index 00000000..4ef98a6a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/ollama.jsonnet @@ -0,0 +1,150 @@ +// Ollama LLM Model Definitions +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemma3:12b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/openai.jsonnet new file mode 100644 index 00000000..75b2670f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/openai.jsonnet @@ -0,0 +1,31 @@ +// OpenAI LLM Model Definitions +// Defines available models and their configurations for OpenAI's platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-4o", + "enum": [ + { + id: "gpt-4o", + description: "GPT-4o (latest)" + }, + { + id: "gpt-4o-mini", + description: "GPT-4o Mini" + }, + { + id: "gpt-4-turbo", + description: "GPT-4 Turbo" + }, + { + id: "gpt-4", + description: "GPT-4" + }, + { + id: "gpt-3.5-turbo", + description: "GPT-3.5 Turbo" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/temperature-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/temperature-param-types.jsonnet new file mode 100644 index 00000000..f5523e88 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/temperature-param-types.jsonnet @@ -0,0 +1,15 @@ +// LLM temperature definitions + +{ + "llm-temperature": { + "type": "float", + "description": "LLM temperature", + "placeholder": 0.3, + "helper": "A floating point number between 0 and 1", + "default": 0.3, + "min": 0.0, + "max": 10.0, + "required": true + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/vertexai.jsonnet new file mode 100644 index 00000000..6d90f8d3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/vertexai.jsonnet @@ -0,0 +1,68 @@ +// VertexAI LLM Model Definitions +// Defines available models and their configurations for Google's VertexAI platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 2.5 models (latest generation) + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-exp", + description: "Gemini 2.0 Flash (experimental)" + }, + + + // Claude models on VertexAI + { + id: "claude-3-5-sonnet@20241022", + description: "Claude 3.5 Sonnet (via VertexAI)" + }, + { + id: "claude-3-5-haiku@20241022", + description: "Claude 3.5 Haiku (via VertexAI)" + }, + { + id: "claude-3-opus@20240229", + description: "Claude 3 Opus (via VertexAI)" + }, + { + id: "claude-3-sonnet@20240229", + description: "Claude 3 Sonnet (via VertexAI)" + }, + { + id: "claude-3-haiku@20240307", + description: "Claude 3 Haiku (via VertexAI)" + }, + + // Llama models on VertexAI + { + id: "llama3-405b-instruct-maas", + description: "Llama 3 405B Instruct (via VertexAI)" + }, + { + id: "llama3-70b-instruct-maas", + description: "Llama 3 70B Instruct (via VertexAI)" + }, + { + id: "llama3-8b-instruct-maas", + description: "Llama 3 8B Instruct (via VertexAI)" + }, + ], + "required": true +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/vllm.jsonnet new file mode 100644 index 00000000..e372a2ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/parameters/vllm.jsonnet @@ -0,0 +1,125 @@ +// vLLM Model Definitions +// Defines available models and their configurations for vLLM +// vLLM works with any HuggingFace model, these are common optimized choices + +{ + "type": "string", + "description": "LLM model to use", + "default": "meta-llama/Llama-3.1-8B-Instruct", + "enum": [ + // Llama 3.1 models + { + id: "meta-llama/Llama-3.1-8B-Instruct", + description: "Llama 3.1 8B Instruct" + }, + { + id: "meta-llama/Llama-3.1-70B-Instruct", + description: "Llama 3.1 70B Instruct" + }, + { + id: "meta-llama/Llama-3.1-405B-Instruct", + description: "Llama 3.1 405B Instruct" + }, + + // Llama 3.2 models + { + id: "meta-llama/Llama-3.2-1B-Instruct", + description: "Llama 3.2 1B Instruct" + }, + { + id: "meta-llama/Llama-3.2-3B-Instruct", + description: "Llama 3.2 3B Instruct" + }, + + // Qwen2.5 models + { + id: "Qwen/Qwen2.5-0.5B-Instruct", + description: "Qwen2.5 0.5B Instruct" + }, + { + id: "Qwen/Qwen2.5-1.5B-Instruct", + description: "Qwen2.5 1.5B Instruct" + }, + { + id: "Qwen/Qwen2.5-3B-Instruct", + description: "Qwen2.5 3B Instruct" + }, + { + id: "Qwen/Qwen2.5-7B-Instruct", + description: "Qwen2.5 7B Instruct" + }, + { + id: "Qwen/Qwen2.5-14B-Instruct", + description: "Qwen2.5 14B Instruct" + }, + { + id: "Qwen/Qwen2.5-32B-Instruct", + description: "Qwen2.5 32B Instruct" + }, + { + id: "Qwen/Qwen2.5-72B-Instruct", + description: "Qwen2.5 72B Instruct" + }, + + // Mistral models + { + id: "mistralai/Mistral-7B-Instruct-v0.3", + description: "Mistral 7B Instruct v0.3" + }, + { + id: "mistralai/Mixtral-8x7B-Instruct-v0.1", + description: "Mixtral 8x7B Instruct" + }, + { + id: "mistralai/Mixtral-8x22B-Instruct-v0.1", + description: "Mixtral 8x22B Instruct" + }, + + // Phi models + { + id: "microsoft/Phi-3.5-mini-instruct", + description: "Phi 3.5 Mini Instruct" + }, + { + id: "microsoft/Phi-4", + description: "Phi 4" + }, + + // Gemma models + { + id: "google/gemma-2-2b-it", + description: "Gemma 2 2B Instruct" + }, + { + id: "google/gemma-2-9b-it", + description: "Gemma 2 9B Instruct" + }, + { + id: "google/gemma-2-27b-it", + description: "Gemma 2 27B Instruct" + }, + + // DeepSeek models + { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", + description: "DeepSeek-R1 Distill Qwen 1.5B" + }, + { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", + description: "DeepSeek-R1 Distill Qwen 7B" + }, + { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + description: "DeepSeek-R1 Distill Qwen 14B" + }, + { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + description: "DeepSeek-R1 Distill Qwen 32B" + }, + { + id: "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + description: "DeepSeek-R1 Distill Llama 70B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/agent-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/agent-prompt.txt new file mode 100644 index 00000000..9155d20f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/agent-prompt.txt @@ -0,0 +1,104 @@ +# ReAct Agent System Prompt + +You are an AI assistant that uses the ReAct (Reasoning + Acting) framework to solve problems through systematic reasoning and tool use. + +## Core Instructions + +For each user query, work through the problem step-by-step using this cycle: +1. **Thought**: Reason about the current situation and determine what you need to do next +2. **Action**: Take ONE specific action using an available tool +3. Wait for **Observation**: The system will provide the result of your action +4. Continue with the next **Thought** based on the observation + +**CRITICAL**: Generate exactly ONE Thought followed by ONE Action, then STOP. Do not generate multiple Thought/Action pairs in a single response. Do not generate Observations yourself - the system will provide them. + +## Response Format + +Use this exact format for each step: + +``` +Thought: [Your reasoning about what to do next - be specific about why this action is needed] +Action: [tool_name] +Args: { + "parameter_name": "value", + "another_parameter": 123, + "list_parameter": ["item1", "item2"] +} +``` + +When you have finished provide the final answer: + +``` +Thought: [Your reasoning about why the process is complete] +Final Answer: [The final answer] +``` + +When providing a final answer, do not provide an Action or Args. + +## Action Format Rules + +1. **Tool Name**: Write "Action: " followed by the exact tool name on its own line +2. **Arguments**: Write "Args: " followed by a valid JSON object containing all parameters +3. **JSON Requirements**: + - Use double quotes for all string keys and values + - Numbers don't need quotes: `"count": 5` + - Booleans: `"enabled": true` or `"enabled": false` + - Arrays: `"items": ["a", "b", "c"]` + - Nested objects: `"config": {"setting": "value"}` + - Null values: `"optional_field": null` +4. **Required Parameters**: Include all required parameters for the tool +5. **No Extra Text**: Don't add explanations or comments within the Action block +1. **Final answer**: Write "Final Answer: " followed by the final answer + +## Available Tools + +{% for tool in tools %}- **{{ tool.name }}**: {{ tool.description }} +{% for arg in tool.arguments %} - Required: `"{{ arg.name }}"` ({{ arg.type }}): {{ arg.description }} +{% endfor %} +{% endfor %} + +## Behavior Rules + +1. **One Step at a Time**: Generate exactly one Thought and one Action, then wait for the system to provide an Observation +2. **Be Specific**: Your Thought should clearly explain why you're taking the specific action +3. **Use Context**: Build on previous Observations to inform your next steps +4. **Error Handling**: If an action fails, reason about the error and try a different approach +5. **Completion**: When you have enough information to fully answer the user's query, generate a final Thought explaining your conclusion, but do not take further actions + +## Error Responses + +If an action fails, you'll see: +``` +Observation: Error: [specific error message] +``` + +When this happens: +- Generate a Thought analyzing what went wrong +- Take a corrective Action with different parameters or a different tool +- If a tool is completely unavailable, explain this limitation in your next Thought + +## Termination + +The conversation ends when: +- You determine you have sufficient information to answer the user's query completely and provide a final answer. +- You encounter an unrecoverable error that prevents task completion +- The system reaches the maximum iteration limit + +## Important Notes + +- **Never generate Observations yourself** - only the system provides these +- **Always validate your JSON** - malformed JSON will cause action failures +- **Stay focused** - each Thought should directly relate to solving the user's query +- **Be efficient** - choose actions that gather the most relevant information for the task + +# Proceed + +Question: {{question}} + +{% for h in history %} +Action: "{{h.action}}" +Args: { +{% for k, v in h.arguments.items() %} "{{k}}": "{{v}}" +{% endfor %}} +Observation: "{{h.observation}}" +{% endfor %} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/cohere.jsonnet new file mode 100644 index 00000000..9541e4c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/cohere.jsonnet @@ -0,0 +1,42 @@ +// For Cohere. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/default-prompts.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/default-prompts.jsonnet new file mode 100644 index 00000000..35dda682 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/default-prompts.jsonnet @@ -0,0 +1,234 @@ + +// Prompt templates. For tidy JSONNET use, don't change these templates +// here, but use over-rides in the prompt directory + +{ + + "system-template":: "You are a helpful assistant.", + + "templates":: { + + "question":: { + "prompt": "{{question}}", + }, + + "extract-definitions":: { + "prompt": "\nStudy the following text and derive definitions for any discovered entities.\nDo not provide definitions for entities whose definitions are incomplete\nor unknown.\nOutput relationships in JSON format as an array of objects with fields:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract will be written as plain text. Do not add markdown formatting\nor headers or prefixes. Do not include null or unknown definitions.\n", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "entity": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "entity", + "definition" + ] + } + } + }, + + "extract-relationships":: { + "prompt": "\nStudy the following text and derive entity relationships. For each\nrelationship, derive the subject, predicate and object of the relationship.\nOutput relationships in JSON format as an array of objects with fields:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: false if the object is a simple data type: name, value or date. true if it is an entity.\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract must be written as plain text. Do not add markdown formatting\nor headers or prefixes.\n", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "object": { + "type": "string" + }, + "object-entity": { + "type": "boolean" + }, + }, + "required": [ + "subject", + "predicate", + "object", + "object-entity" + ] + } + } + }, + + "extract-topics":: { + "prompt": "You are a helpful assistant that performs information extraction tasks for a provided text.\nRead the provided text. You will identify topics and their definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The JSON response shall use the following structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "topic": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "topic", + "definition" + ] + } + } + }, + + "extract-rows":: { + "prompt": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "response-type": "json", + }, + + "kg-prompt":: { + "prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "response-type": "text", + }, + + "document-prompt":: { + "prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "response-type": "text", + }, + + "agent-react":: { + "prompt": importstr "agent-prompt.txt", + "response-type": "text" + }, + + "agent-kg-extract":: { + "prompt": "Analyze the following text and extract both entity definitions and relationships. Return the results as JSON with 'definitions' and 'relationships' arrays.\n\nFor definitions, extract entities and their explanations or descriptions.\nFor relationships, extract subject-predicate-object triples where subjects and objects are entities, and predicates are relationship types.\n\nText: {{text}}\n\nReturn JSON only, no other text. Use this exact format:\n{\n \"definitions\": [\n {\n \"entity\": \"entity_name\",\n \"definition\": \"definition_text\"\n }\n ],\n \"relationships\": [\n {\n \"subject\": \"subject_entity\",\n \"predicate\": \"relationship_type\",\n \"object\": \"object_entity_or_literal\",\n \"object-entity\": true\n }\n ]\n}\n", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "definitions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "entity": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "entity", + "definition" + ] + } + }, + "relationships": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "object": { + "type": "string" + }, + "object-entity": { + "type": "boolean" + } + }, + "required": [ + "subject", + "predicate", + "object" + ] + } + } + }, + "required": [ + "definitions", + "relationships" + ] + } + }, + + "schema-selection":: { + "prompt": importstr "schema-selection.txt", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "An array of schema names that are relevant to answering the given question" + } + }, + + "graphql-generation":: { + "prompt": importstr "graphql-generation.txt", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The GraphQL query string generated to answer the question" + }, + "variables": { + "type": "object", + "description": "Object containing any GraphQL variables needed for the query", + "additionalProperties": true + }, + "confidence": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "description": "Float between 0.0-1.0 indicating confidence in the generated query" + } + }, + "required": ["query", "variables", "confidence"], + "additionalProperties": false + } + }, + + "diagnose-structured-data":: { + "prompt": importstr "diagnose-structured-data.txt", + "response-type": "json", + }, + + "diagnose-xml":: { + "prompt": importstr "diagnose-xml.txt", + "response-type": "json", + }, + "diagnose-json":: { + "prompt": importstr "diagnose-json.txt", + "response-type": "json", + }, + "diagnose-csv":: { + "prompt": importstr "diagnose-csv.txt", + "response-type": "json", + }, + + "extract-with-ontologies":: { + "prompt": importstr "ontology-prompt.txt", + "response-type": "json", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-csv.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-csv.txt new file mode 100644 index 00000000..ad9725ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-csv.txt @@ -0,0 +1,370 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in CSV processing and delimiter-separated value formats. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured CSV data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## CSV Processing Expertise + +When working with CSV data, you must: + +1. **Analyze CSV Structure** - Examine headers, delimiters, quoting, and data patterns +2. **Identify Column Mappings** - Map source column names to target fields +3. **Handle Complex CSV Patterns** - Support various CSV formats including: + - Standard comma-separated values: `name,age,city` + - Alternative delimiters: tab-separated (TSV), pipe-separated, semicolon-separated + - Quoted fields with embedded delimiters: `"Last, First",25,"New York, NY"` + - Headers with spaces or special characters: `"Customer Name","Order Date","Total Amount"` + - Files with or without headers + - Multi-line fields with embedded newlines + +## CSV Format Configuration Guidelines + +For CSV format configurations, use these patterns: + +**Basic CSV Configuration:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "skip_rows": 0 + } + } +} +``` + +**Advanced CSV Options:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": "\t", // Tab-separated + "quote_char": "\"", + "escape_char": "\\", + "has_header": true, + "skip_rows": 2, // Skip metadata rows + "null_values": ["", "NULL", "N/A"], + "trim_whitespace": true, + "skip_blank_lines": true + } + } +} +``` + +**CRITICAL: Source Field Names in Mappings** + +When processing CSV files, the parser uses column headers (if present) or generates column indices as field names. Your source field names in mappings must match exactly: + +**CORRECT Example with Headers:** +CSV file: +```csv +Customer Name,Order Date,Total Amount,Status +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "Customer Name": "John Smith", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Customer Name", // βœ… Correct - matches header exactly + "source_field": "Order Date", // βœ… Correct - matches header exactly + "source_field": "Total Amount", // βœ… Correct - matches header exactly + "source_field": "Status" // βœ… Correct - matches header exactly +} +``` + +**CORRECT Example without Headers:** +CSV file without headers uses column indices: +```csv +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "0": "John Smith", + "1": "2024-01-15", + "2": "1000.50", + "3": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "0", // βœ… Correct - first column + "source_field": "1", // βœ… Correct - second column + "source_field": "2", // βœ… Correct - third column + "source_field": "3" // βœ… Correct - fourth column +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - Delimiter character (comma, tab, pipe, semicolon, etc.) + - Quote character and escape character + - **For CSV**: Does the file have headers? Sample structure + - Text encoding (UTF-8, Windows-1252, etc.) + - Any rows to skip (metadata, blank lines) + - How null/empty values are represented + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source column β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Date/time format conversions + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + - Row-level validation rules + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## CSV Structure Analysis + +When presented with CSV data, analyze: + +1. **Delimiter Detection**: What character separates the fields? +2. **Header Presence**: Does the first row contain column names? +3. **Quote Pattern**: Are fields quoted? What quote character is used? +4. **Data Types**: What types are present in each column? +5. **Null Representation**: How are empty/null values represented? +6. **Special Characters**: Are there embedded commas, quotes, or newlines? +7. **Encoding Issues**: Are there any character encoding problems? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + // CSV-specific parsing options + // delimiter, quote_char, has_header, skip_rows, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `split`, `strip_quotes` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_number`, `parse_currency` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `clean_whitespace`, `normalize_encoding` + +**Date/Time Operations:** +- `parse_date`, `format_date`, `date_component` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `numeric_range`, `date_range` + +## CSV-Specific Best Practices + +1. **Detect delimiters accurately** - test with sample data to confirm delimiter +2. **Handle quoted fields properly** - account for embedded delimiters and quotes +3. **Trim whitespace consistently** - decide whether to preserve or remove extra spaces +4. **Validate data types early** - catch type conversion errors at the field level +5. **Handle empty values explicitly** - distinguish between empty strings and nulls +6. **Account for encoding issues** - especially with international characters +7. **Validate row structure** - ensure consistent column counts across rows + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle CSV-specific edge cases** like malformed quotes, inconsistent delimiters +10. **Test with sample data** to validate parsing configuration + +## Complete CSV Example + +Given this CSV structure: +```csv +Customer Name,Order Date,Total Amount,Currency,Status +"Smith, John",2024-01-15,1000.50,USD,Active +"Doe, Jane",2024-01-16,750.25,EUR,Pending +"Johnson, Bob",2024-01-17,,USD,Cancelled +``` + +The parser will: +1. Use `delimiter: ","` and `quote_char: "\""` to parse fields correctly +2. Use `has_header: true` to treat first row as column names +3. Create this parsed data structure for each record: + ```json + { + "Customer Name": "Smith, John", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Currency": "USD", + "Status": "Active" + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "trim_whitespace": true, + "null_values": ["", "NULL"] + } + }, + "mappings": [ + { + "source_field": "Customer Name", // βœ… Matches header exactly + "target_field": "customer_name", + "transforms": [{"type": "trim"}] + }, + { + "source_field": "Order Date", // βœ… Matches header exactly + "target_field": "order_date", + "transforms": [{"type": "to_date", "format": "YYYY-MM-DD"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "Total Amount", // βœ… Matches header exactly + "target_field": "amount", + "transforms": [ + {"type": "to_float"}, + {"type": "default", "value": 0.0} + ] + }, + { + "source_field": "Currency", // βœ… Matches header exactly + "target_field": "currency_code", + "transforms": [{"type": "upper"}], + "validation": [{"type": "in_list", "values": ["USD", "EUR", "GBP"]}] + }, + { + "source_field": "Status", // βœ… Matches header exactly + "target_field": "order_status", + "transforms": [{"type": "lower"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the column headers exactly, or use column indices (0, 1, 2, etc.) for files without headers.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the CSV structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to delimiter detection, header identification, quoting patterns, and data type inference: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-json.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-json.txt new file mode 100644 index 00000000..2fe6341d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-json.txt @@ -0,0 +1,327 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in JSON processing and JSONPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured JSON data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## JSON Processing Expertise + +When working with JSON data, you must: + +1. **Analyze JSON Structure** - Examine the hierarchy, array patterns, and object nesting +2. **Generate Proper JSONPath Expressions** - Create efficient JSONPath selectors for record extraction +3. **Handle Complex JSON Patterns** - Support various JSON formats including: + - Array of objects: `[{"name": "John", "age": 30}, {...}]` + - Nested object arrays: `{"data": {"records": [{"id": 1}, {...}]}}` + - Mixed hierarchies with both arrays and nested objects + - Single object records: `{"record": {"field1": "value1"}}` + +## JSONPath Expression Guidelines + +For JSON format configurations, use these JSONPath patterns: + +**Record Path Examples:** +- Root array: `$[*]` (for arrays at the root level) +- Nested arrays: `$.data.records[*]` or `$.response.items[*]` +- Single object: `$.record` (when there's one record per file) +- Deep nesting: `$.data.results.items[*]` + +**Field Access Patterns:** +- Direct properties: Use property names directly in mappings +- Nested properties: Use dot notation like `address.street` or `contact.email` +- Array elements: Use bracket notation like `tags[0]` for first element + +**CRITICAL: Source Field Names in Mappings** + +When processing JSON, the parser creates a flat or nested dictionary based on the record structure. Your source field names in mappings must match the actual property names in the parsed records: + +**CORRECT Example:** +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "metadata": { + "source": "UN", + "year": 2024 + } +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches property name + "source_field": "Trade (USD)", // βœ… Correct - matches property name + "source_field": "metadata.source", // βœ… Correct - nested property access + "source_field": "metadata.year" // βœ… Correct - nested property access +} +``` + +**JSON Format Configuration Template:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$[*]", // JSONPath to extract records + "flatten_nested": true, // Whether to flatten nested objects + "array_handling": "expand" // How to handle arrays: expand, first, concat + } + } +} +``` + +**Alternative JSON Options:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.items[*]", // For nested array structures + "flatten_nested": false, // Keep nested structure + "null_value_handling": "skip" // skip, empty_string, or preserve + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - JSON structure type (array of objects, nested objects, single records) + - **For JSON**: Sample structure, nesting patterns, array locations + - Sample data or field descriptions + - Any format-specific details (encoding, special null handling, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Nested object flattening requirements + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or null values + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## JSON Structure Analysis + +When presented with JSON data, analyze: + +1. **Root Structure**: Is it an array, object, or nested structure? +2. **Record Location**: Where are individual records located in the hierarchy? +3. **Field Pattern**: How are field names and values structured? + - Direct properties: `{"name": "John"}` + - Nested objects: `{"contact": {"email": "john@example.com"}}` + - Arrays: `{"tags": ["red", "blue"]}` + - Mixed types: `{"data": [{"id": 1, "details": {"name": "John"}}]}` +4. **Data Types**: What types are present (strings, numbers, booleans, nulls, arrays, objects)? +5. **Hierarchy Depth**: How deeply nested are the records and fields? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + // JSON-specific parsing options + // record_path (JSONPath), flatten_nested, array_handling, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `flatten_object`, `extract_array_element`, `join_array` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## JSON-Specific Best Practices + +1. **Use efficient JSONPath expressions** - Prefer specific paths over broad searches +2. **Handle nested objects appropriately** - decide whether to flatten or preserve structure +3. **Consider array handling strategies** - expand arrays to multiple records or extract specific elements +4. **Account for null vs undefined** values in field mappings +5. **Handle mixed data types** within the same field across records +6. **Use appropriate flattening** for deeply nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle JSON-specific edge cases** like empty arrays, null objects, mixed types + +## Complete JSON Example + +Given this JSON structure: +```json +{ + "data": { + "records": [ + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata": { + "source": "World Bank", + "confidence": 0.95 + } + } + ] + } +} +``` + +The parser will: +1. Use `record_path: "$.data.records[*]"` to extract record objects from the array +2. Create this parsed data structure for each record: + ```json + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata.source": "World Bank", // If flattened + "metadata.confidence": 0.95 // If flattened + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.records[*]", + "flatten_nested": true, + "array_handling": "expand" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches property name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches property name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches property name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + }, + { + "source_field": "metadata.source", // βœ… Flattened nested field + "target_field": "data_source" + } + ] +} +``` + +**KEY RULE: source_field names must match the actual property names in the parsed JSON records, using dot notation for nested properties when flattened.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the JSON structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to JSON hierarchy, object patterns, array structures, and generate appropriate JSONPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-structured-data.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-structured-data.txt new file mode 100644 index 00000000..84c4b8be --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-structured-data.txt @@ -0,0 +1,309 @@ + +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` (will be converted to relative paths automatically) +- With namespaces: `//ns:record` or `//soap:Body/data/record` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name + +**CRITICAL: Source Field Names in Mappings** + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary. Your source field names in mappings must match these extracted names: + +**CORRECT Example:** +```xml +Albania +1000.50 +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50" +} +``` + +So your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)" // βœ… Correct - matches parsed field name +} +``` + +**INCORRECT Example:** +```json +{ + "source_field": "Field[@name='Country or Area']", // ❌ Wrong - XPath not needed here + "source_field": "field[@name='Trade (USD)']" // ❌ Wrong - XPath not needed here +} +``` + +**XML Format Configuration Template:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name" // For value pattern + } + } +} +``` + +**Alternative XML Options:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // Direct element-based records + // No field_attribute needed for standard XML + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - File type (CSV, JSON, XML, Excel, fixed-width, etc.) + - **For XML**: Sample structure, namespace prefixes, record element patterns + - Sample data or field descriptions + - Any format-specific details (delimiters, encoding, namespaces, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Record Container**: Where are individual records located? +3. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns +4. **Namespaces**: Are there any namespace prefixes? +5. **Hierarchy Depth**: How deeply nested are the records? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "[csv|json|xml|fixed-width|excel]", + "encoding": "utf-8", + "options": { + // Format-specific parsing options + // For XML: record_path (XPath), field_attribute (if applicable) + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches +2. **Handle namespace prefixes** when present +3. **Identify field attribute patterns** correctly +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes + +## Complete XML Example + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "/ROOT/data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the extracted field names, NOT the XML element structure.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, and generate appropriate XPath expressions: + +{{sample}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-xml.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-xml.txt new file mode 100644 index 00000000..a3d8efa4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/diagnose-xml.txt @@ -0,0 +1,453 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for XML data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured XML data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + - CDATA sections and text nodes + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` +- With namespaces: `//ns:record` or `//soap:Body/data/record` +- Attribute-filtered: `//record[@type='customer']` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name like `field_attribute: "id"` or `field_attribute: "type"` + +## CRITICAL: Source Field Names in Mappings + +The behavior depends on whether you use `field_attribute`: + +### With field_attribute (Attribute-Based Fields) + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary: + +**Example:** +```xml + + Albania + 1000.50 + Active + +``` + +Parser configuration: +```json +{ + "record_path": "//record", + "field_attribute": "name" +} +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)", // βœ… Correct - matches parsed field name + "source_field": "Status" // βœ… Correct - matches parsed field name +} +``` + +### Without field_attribute (Element-Based Fields) + +When NOT using `field_attribute`, use direct element names or XPath expressions: + +**Example:** +```xml + + Albania + 1000.50 + Active + + UN + 2024 + + +``` + +Parser configuration: +```json +{ + "record_path": "//record" + // No field_attribute specified +} +``` + +Your mappings should use: +```json +{ + "source_field": "country", // βœ… Direct element name + "source_field": "trade_amount", // βœ… Direct element name + "source_field": "metadata/source", // βœ… Nested element path + "source_field": "metadata/year" // βœ… Nested element path +} +``` + +## XML Format Configuration Templates + +**For Attribute-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name", // Extract field names from 'name' attribute + "namespace_prefixes": { // Optional: define namespaces + "ns": "http://example.com/namespace" + } + } + } +} +``` + +**For Element-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // XPath to find record elements + "preserve_namespaces": true, // Optional: keep namespace info + "ignore_attributes": false // Optional: include element attributes + } + } +} +``` + +**For Complex XML with Namespaces:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//soap:Body//data:record", + "namespace_prefixes": { + "soap": "http://schemas.xmlsoap.org/soap/envelope/", + "data": "http://example.com/data" + }, + "field_attribute": "name" + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **XML Structure Details** + - Sample XML structure or schema + - Root element and record location + - Field organization (elements vs attributes) + - Namespace declarations and prefixes + - Text encoding (UTF-8, ISO-8859-1, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Namespaces**: Are there namespace declarations? What prefixes are used? +3. **Record Container**: Where are individual records located in the hierarchy? +4. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns: `John` +5. **Data Location**: Are values in element text, attributes, or both? +6. **Hierarchy Depth**: How deeply nested are the records? +7. **Special Content**: Any CDATA sections, mixed content, or special characters? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + // XML-specific parsing options + // record_path (XPath), field_attribute (if applicable), namespace_prefixes + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `strip_cdata`, `decode_html_entities` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_xml_date`, `parse_iso_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `extract_attribute`, `get_text_content` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `valid_xml_name`, `namespace_check` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches like `//` when possible +2. **Handle namespace prefixes** when present - define them in `namespace_prefixes` +3. **Identify field attribute patterns** correctly - determine if using `field_attribute` +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures +7. **Handle CDATA sections** and special characters properly +8. **Preserve or normalize whitespace** as appropriate +9. **Consider XML schema validation** if XSD is available + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle XML-specific edge cases** like empty elements, mixed content, processing instructions + +## Complete XML Examples + +### Example 1: Attribute-Based Fields + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "//data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +### Example 2: Element-Based Fields + +Given this XML structure: +```xml + + + John Smith + john@example.com +
+ 123 Main St + New York +
+ + + 456 + 100.50 + + +
+
+``` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer" + } + }, + "mappings": [ + { + "source_field": "name", // βœ… Direct element name + "target_field": "customer_name" + }, + { + "source_field": "email", // βœ… Direct element name + "target_field": "email_address" + }, + { + "source_field": "address/street", // βœ… Nested element path + "target_field": "street_address" + }, + { + "source_field": "address/city", // βœ… Nested element path + "target_field": "city" + }, + { + "source_field": "@id", // βœ… Attribute reference + "target_field": "customer_id", + "transforms": [{"type": "to_int"}] + } + ] +} +``` + +**KEY RULES:** +- With `field_attribute`: source_field names must match the extracted attribute values +- Without `field_attribute`: use direct element names, nested paths, or XPath expressions +- Use `@attribute` notation for XML element attributes + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, namespace usage, and generate appropriate XPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/gemini.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/gemini.jsonnet new file mode 100644 index 00000000..b9a1e0c0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/gemini.jsonnet @@ -0,0 +1,42 @@ +// For VertexAI Gemini. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/graphql-generation.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/graphql-generation.txt new file mode 100644 index 00000000..58af61dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/graphql-generation.txt @@ -0,0 +1,59 @@ +You are a GraphQL query generation expert. Given a natural language question and provided database schemas, generate a valid GraphQL query embedded in valid JSON. + +## Question: +{{ question }} + +## Provided Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %}- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif %}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif %}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ field.enum_values|join(', ') }}]{% endif %} +{% endfor %} +{% endfor %} + +## GraphQL Query Rules: +1. Use the schema names as GraphQL query fields (e.g., `customers`, `orders`) +2. Apply filters using the `where` parameter with nested filter objects +3. Available filter operators per field type: + - String fields: `eq`, `contains`, `startsWith`, `endsWith`, `in`, `not`, `not_in` + - Integer/Float fields: `eq`, `gt`, `gte`, `lt`, `lte`, `in`, `not`, `not_in` +4. Use `order_by` for sorting (field name as string) +5. Use `direction` for sort direction: `ASC` or `DESC` +6. Use `limit` to restrict number of results +7. Select specific fields in the query body + +## Task Instructions: +1. Analyze the question to identify: + - What data to retrieve (which fields to select) + - What filters to apply (where conditions) + - What sorting is needed (order_by, direction) + - How many results (limit) + +2. Generate a GraphQL query that: + - Uses only the provided schema names and field names + - Applies appropriate filters based on the question + - Selects relevant fields for the response + - Includes reasonable limits (default 100 if not specified) + +3. If variables are needed, include them in the response + +## Output Format: +Return ONLY a valid JSON object (no markdown, no code blocks) with these fields: +- "query": the GraphQL query string +- "variables": object with any GraphQL variables (empty object if none) +- "confidence": float between 0.0-1.0 indicating confidence in the query + +Important: Return raw JSON only, with no markdown formatting, no code blocks, and no backticks. + +## Examples: + +Question: "Show me customers from California" +{"query": "query { customers(where: {state: {eq: \"California\"}}, limit: 100) { customer_id name email state } }", "variables": {}, "confidence": 0.92} + +Question: "Top 10 products by price" +{"query": "query { products(order_by: \"price\", direction: DESC, limit: 10) { product_id name price category } }", "variables": {}, "confidence": 0.88} + +Question: "Recent orders over $100" +{"query": "query { orders(where: {total_amount: {gt: 100}, order_date: {gte: \"2024-01-01\"}}, order_by: \"order_date\", direction: DESC, limit: 50) { order_id customer_id total_amount order_date status } }", "variables": {}, "confidence": 0.96} + +Now generate the GraphQL query for the question above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/mixtral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/mixtral.jsonnet new file mode 100644 index 00000000..cd56e7ef --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/mixtral.jsonnet @@ -0,0 +1,42 @@ +// For Mixtral. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/ontology-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/ontology-prompt.txt new file mode 100644 index 00000000..6be255b7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/ontology-prompt.txt @@ -0,0 +1,54 @@ +You are a knowledge extraction expert. Extract structured triples from text using ONLY the provided ontology elements. + +## Ontology Classes: + +{% for class_id, class_def in classes.items() %} +- **{{class_id}}**{% if class_def.subclass_of %} (subclass of {{class_def.subclass_of}}){% endif %}{% if class_def.comment %}: {{class_def.comment}}{% endif %} +{% endfor %} + +## Object Properties (connect entities): + +{% for prop_id, prop_def in object_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Datatype Properties (entity attributes): + +{% for prop_id, prop_def in datatype_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Text to Analyze: + +{{text}} + +## Extraction Rules: + +1. Only use classes defined above for entity types +2. Only use properties defined above for relationships and attributes +3. Respect domain and range constraints where specified +4. For class instances, use `rdf:type` as the predicate +5. Include `rdfs:label` for new entities to provide human-readable names +6. Extract all relevant triples that can be inferred from the text +7. Use entity URIs or meaningful identifiers as subjects/objects + +## Output Format: + +Return ONLY a valid JSON array (no markdown, no code blocks) containing objects with these fields: +- "subject": the subject entity (URI or identifier) +- "predicate": the property (from ontology or rdf:type/rdfs:label) +- "object": the object entity or literal value + +Important: Return raw JSON only, with no markdown formatting, no code blocks, and no backticks. + +## Example Output: + +[ + {"subject": "recipe:cornish-pasty", "predicate": "rdf:type", "object": "Recipe"}, + {"subject": "recipe:cornish-pasty", "predicate": "rdfs:label", "object": "Cornish Pasty"}, + {"subject": "recipe:cornish-pasty", "predicate": "has_ingredient", "object": "ingredient:flour"}, + {"subject": "ingredient:flour", "predicate": "rdf:type", "object": "Ingredient"}, + {"subject": "ingredient:flour", "predicate": "rdfs:label", "object": "plain flour"} +] + +Now extract triples from the text above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/openai.jsonnet new file mode 100644 index 00000000..5d232337 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/openai.jsonnet @@ -0,0 +1,42 @@ +// For OpenAI LLMs. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/schema-selection.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/schema-selection.txt new file mode 100644 index 00000000..39b180e5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/schema-selection.txt @@ -0,0 +1,25 @@ +You are a database schema selection expert. Given a natural language question and available +database schemas, your job is to identify which schemas are most relevant to answer the question. + +## Available Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}): {{ field.description }} +{% endfor %} + +{% endfor %} + +## Question: +{{ question }} + +## Instructions: +1. Analyze the question to understand what data is being requested +2. Examine each schema to understand what data it contains +3. Select ONLY the schemas that are directly relevant to answering the question +4. Return your answer as a JSON array of schema names + +## Response Format: +Return ONLY a JSON array of schema names, nothing else. +Example: ["customers", "orders", "products"] diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/slm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/slm.jsonnet new file mode 100644 index 00000000..48eb96d0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/prompts/slm.jsonnet @@ -0,0 +1,44 @@ +// For SLM. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/cassandra.jsonnet new file mode 100644 index 00000000..2a9d6d7a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/cassandra.jsonnet @@ -0,0 +1,40 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "cassandra" +: { + + create:: function(engine) + + local vol = engine.volume("cassandra").with_size("20G"); + + local container = + engine.container("cassandra") + .with_image(images.cassandra) + .with_environment({ + JVM_OPTS: "-Xms300M -Xmx300M -Dcassandra.skip_wait_for_gossip_to_settle=0", + }) + .with_limits("1.0", "1000M") + .with_reservations("0.5", "1000M") + .with_port(9042, 9042, "cassandra") + .with_volume_mount(vol, "/var/lib/cassandra"); + + local containerSet = engine.containers( + "cassandra", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9042, 9042, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/falkordb.jsonnet new file mode 100644 index 00000000..78509a43 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/falkordb.jsonnet @@ -0,0 +1,39 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "falkordb" +: { + + create:: function(engine) + + local vol = engine.volume("falkordb").with_size("20G"); + + local container = + engine.container("falkordb") + .with_image(images.falkordb) + .with_limits("1.0", "768M") + .with_reservations("0.5", "768M") + .with_port(6379, 6379, "api") + .with_port(3010, 3000, "ui") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "falkordb", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6379, 6379, "api") + .with_port(3010, 3010, "ui"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/memgraph.jsonnet new file mode 100644 index 00000000..70ad127a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/memgraph.jsonnet @@ -0,0 +1,71 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "memgraph" +: { + + create:: function(engine) + + local vol = engine.volume("memgraph").with_size("20G"); + + local container = + engine.container("memgraph") + .with_image(images.memgraph_mage) + .with_environment({ + MEMGRAPH: "--storage-properties-on-edges=true --storage-enable-edges-metadata=true" + }) + .with_limits("1.0", "1000M") + .with_reservations("0.5", "1000M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/var/lib/memgraph"); + + local containerSet = engine.containers( + "memgraph", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + "memgraph-lab" +: { + + create:: function(engine) + + local container = + engine.container("lab") + .with_image(images.memgraph_lab) + .with_environment({ + QUICK_CONNECT_MG_HOST: "memgraph", + QUICK_CONNECT_MG_PORT: "7687", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3010, 3000, "http"); + + local containerSet = engine.containers( + "lab", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3010, 3010, "http"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/milvus.jsonnet new file mode 100644 index 00000000..1c3e3734 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/milvus.jsonnet @@ -0,0 +1,90 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local minio = import "stores/minio.jsonnet"; + +minio { + + etcd +: { + + create:: function(engine) + + local vol = engine.volume("etcd").with_size("20G"); + + local container = + engine.container("etcd") + .with_image(images.etcd) + .with_command([ + "etcd", + "-advertise-client-urls=http://127.0.0.1:2379", + "-listen-client-urls", + "http://0.0.0.0:2379", + "--data-dir", + "/etcd", + ]) + .with_environment({ + ETCD_AUTO_COMPACTION_MODE: "revision", + ETCD_AUTO_COMPACTION_RETENTION: "1000", + ETCD_QUOTA_BACKEND_BYTES: "4294967296", + ETCD_SNAPSHOT_COUNT: "50000" + }) + .with_limits("1.0", "128M") + .with_reservations("0.25", "128M") + .with_port(2379, 2379, "api") + .with_volume_mount(vol, "/etcd"); + + local containerSet = engine.containers( + "etcd", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(2379, 2379, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + milvus +: { + + create:: function(engine) + + local vol = engine.volume("milvus").with_size("20G"); + + local container = + engine.container("milvus") + .with_image(images.milvus) + .with_command([ + "milvus", "run", "standalone" + ]) + .with_environment({ + ETCD_ENDPOINTS: "etcd:2379", + MINIO_ADDRESS: "minio:9000", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2") + .with_volume_mount(vol, "/var/lib/milvus"); + + local containerSet = engine.containers( + "milvus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/minio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/minio.jsonnet new file mode 100644 index 00000000..8a6aa149 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/minio.jsonnet @@ -0,0 +1,49 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + minio +: { + + create:: function(engine) + + local vol = engine.volume("minio-data").with_size("20G"); + + local container = + engine.container("minio") + .with_image(images.minio) + .with_command([ + "minio", + "server", + "/minio_data", + "--console-address", + ":9001", + ]) + .with_environment({ + MINIO_ROOT_USER: "minioadmin", + MINIO_ROOT_PASSWORD: "minioadmin", + }) + .with_limits("0.5", "128M") + .with_reservations("0.25", "128M") + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console") + .with_volume_mount(vol, "/minio_data"); + + local containerSet = engine.containers( + "minio", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/neo4j.jsonnet new file mode 100644 index 00000000..3a8bb783 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/neo4j.jsonnet @@ -0,0 +1,47 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "neo4j" +: { + + create:: function(engine) + + local vol = engine.volume("neo4j").with_size("20G"); + + local container = + engine.container("neo4j") + .with_image(images.neo4j) + .with_environment({ + NEO4J_AUTH: "neo4j/password", + NEO4J_server_memory_pagecache_size: "512m", + NEO4J_server_memory_heap_max__size: "512m", + // NEO4J_server_bolt_listen__address: "0.0.0.0:7687", + // NEO4J_server_default__listen__address: "0.0.0.0", + // NEO4J_server_http_listen__address: "0.0.0.0:7474", + }) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1536M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "neo4j", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/pinecone.jsonnet new file mode 100644 index 00000000..2bef70fb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/pinecone.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "pinecone" +: { + + create:: function(engine) + + local container = + engine.container("pinecone") + .with_image(images.pinecone) + .with_environment({ + + PORT: "5080", + + INDEX_TYPE: "serverless", + + // Dimension is fixed, 384 is right for miniLM + // sentence embedding + DIMENSION: 384, + + METRIC: "cosine" + + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(5080, 5080, "api"); + + local containerSet = engine.containers( + "pinecone", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(5080, 5080, "api"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/qdrant.jsonnet new file mode 100644 index 00000000..9e807632 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/stores/qdrant.jsonnet @@ -0,0 +1,39 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "qdrant" +: { + + create:: function(engine) + + local vol = engine.volume("qdrant").with_size("20G"); + + local container = + engine.container("qdrant") + .with_image(images.qdrant) + .with_limits("1.0", "1024M") + .with_reservations("0.5", "1024M") + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2") + .with_volume_mount(vol, "/qdrant/storage"); + + local containerSet = engine.containers( + "qdrant", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/trustgraph-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/trustgraph-config.jsonnet new file mode 100644 index 00000000..1cfd3002 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/trustgraph-config.jsonnet @@ -0,0 +1,80 @@ +// TrustGraph Main Configuration +// Clean, modular composition of TrustGraph configuration +// Uses specialized modules for different aspects of config building + +// Import dependencies +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local default_prompts = import "prompts/default-prompts.jsonnet"; +local token_costs = import "values/token-costs.jsonnet"; +local flow_classes = import "flows/flow-classes.jsonnet"; +local config_composer = import "config/config-composer.jsonnet"; +local interface_descriptions = import "config/interface-descriptions.jsonnet"; +local tools = import "config/tools.jsonnet"; +local temperature_params = import "parameters/temperature-param-types.jsonnet"; +local chunking_params = import "parameters/chunking-param-types.jsonnet"; + +// Main configuration object +local configuration = { + + // Prompt templates + prompts:: default_prompts, + + // Tool definitions + tools:: tools, + + // MCP configuration + mcp:: {}, + + // Flow classes reference + "flow-classes":: flow_classes, + + // LLM model parameters + "llm-models" +:: {}, + + // Embeddings model parameters + "embeddings-models" +:: {}, + + // Default model and flow parameters + flow_init_parameters:: { + "llm-model": $["llm-models"].default, + "llm-rag-model": $["llm-models"].default, + "llm-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "llm-rag-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "chunk-size": std.toString($["parameter-types"]["chunk-size"].default), + "chunk-overlap": std.toString($["parameter-types"]["chunk-overlap"].default), + "embeddings-model": $["embeddings-models"].default, + }, + + // Interface descriptions for external endpoints + "interface-descriptions":: interface_descriptions, + + // Parameter type definitions + "parameter-types":: { + "llm-model": $["llm-models"], + "embeddings-model": $["embeddings-models"], + } + chunking_params + temperature_params, + + // Token costs + "token-costs":: token_costs, + + // Build the complete configuration using the composer + configuration:: config_composer.build({ + flow_classes: $["flow-classes"], + default_flow_class: "everything", + default_flow_id: "default", + flow_init_parameters: $["flow_init_parameters"], + prompts: $["prompts"], + tools: $["tools"], + mcp: $["mcp"], + interface_descriptions: $["interface-descriptions"], + parameter_types: $["parameter-types"], + token_costs: $["token-costs"], + }), + +} + default_prompts; + +// Export the final configuration +configuration \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/util/decode-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/util/decode-config.jsonnet new file mode 100644 index 00000000..503b5b6b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/util/decode-config.jsonnet @@ -0,0 +1,31 @@ + +local components = import "components.jsonnet"; + +local apply = function(p, components) + + local base = { + + with:: function(k, v) self + { + [k]:: v + }, + + with_params:: function(pars) + self + std.foldl( + function(obj, par) obj.with(par.key, par.value), + std.objectKeysValues(pars), + self + ), + + }; + + local component = base + components[p.name]; + + component.with_params(p.parameters); + +local decode = function(config) + local add = function(state, c) state + apply(c, components); + local patterns = std.foldl(add, config, {}); + patterns; + +decode + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/values/images.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/values/images.jsonnet new file mode 100644 index 00000000..5cf21ee1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/values/images.jsonnet @@ -0,0 +1,31 @@ +local version = import "version.jsonnet"; +{ + cassandra: "docker.io/cassandra:4.1.10", + neo4j: "docker.io/neo4j:2025.08.0-community-bullseye", + pulsar: "docker.io/apachepulsar/pulsar:4.1.0", + pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.4.0", + etcd: "quay.io/coreos/etcd:v3.6.4", + minio: "docker.io/minio/minio:RELEASE.2025-09-07T16-13-09Z", + milvus: "docker.io/milvusdb/milvus:v2.5.17", + prometheus: "docker.io/prom/prometheus:v3.5.0", + grafana: "docker.io/grafana/grafana:12.1.1", + trustgraph_base: "docker.io/trustgraph/trustgraph-base:" + version, + trustgraph_flow: "docker.io/trustgraph/trustgraph-flow:" + version, + trustgraph_ocr: "docker.io/trustgraph/trustgraph-ocr:" + version, + trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version, + trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version, + trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version, + trustgraph_mcp: "docker.io/trustgraph/trustgraph-mcp:" + version, + qdrant: "docker.io/qdrant/qdrant:v1.15.4", + memgraph_mage: "docker.io/memgraph/memgraph-mage:3.5", + memgraph_lab: "docker.io/memgraph/lab:3.5.0", + falkordb: "docker.io/falkordb/falkordb:v4.12.5", + "workbench-ui": "docker.io/trustgraph/workbench-ui:1.4.2", + "ddg-mcp-server": "docker.io/trustgraph/ddg-mcp-server:0.1.0", + "tgi-service-intel-xpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-xpu", + "tgi-service-cpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-cpu", + "tgi-service-gaudi": "ghcr.io/huggingface/text-generation-inference:sha-f140440-gaudi", + "vllm-service-intel-xpu": "docker.io/intel/vllm:0.8.0-xpu", + "vllm-service-gaudi": "docker.io/trustgraph/vllm-hpu:027f5645", + "vllm-service-nvidia": "docker.io/vllm/vllm-openai:latest", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/values/token-costs.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/values/token-costs.jsonnet new file mode 100644 index 00000000..84c70373 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/values/token-costs.jsonnet @@ -0,0 +1,102 @@ +{ + "mistral.mistral-large-2407-v1:0": { + "model_name": "mistral.mistral-large-2407-v1:0", + "input_price": 0.000004, + "output_price": 0.000012 + }, + "meta.llama3-1-405b-instruct-v1:0": { + "model_name": "meta.llama3-1-405b-instruct-v1:0", + "input_price": 0.00000532, + "output_price": 0.000016 + }, + "mistral.mixtral-8x7b-instruct-v0:1": { + "model_name": "mistral.mixtral-8x7b-instruct-v0:1", + "input_price": 0.00000045, + "output_price": 0.0000007 + }, + "meta.llama3-1-70b-instruct-v1:0": { + "model_name": "meta.llama3-1-70b-instruct-v1:0", + "input_price": 0.00000099, + "output_price": 0.00000099 + }, + "meta.llama3-1-8b-instruct-v1:0": { + "model_name": "meta.llama3-1-8b-instruct-v1:0", + "input_price": 0.00000022, + "output_price": 0.00000022 + }, + "anthropic.claude-3-haiku-20240307-v1:0": { + "model_name": "anthropic.claude-3-haiku-20240307-v1:0", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "anthropic.claude-3-5-sonnet-20240620-v1:0": { + "model_name": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "cohere.command-r-plus-v1:0": { + "model_name": "cohere.command-r-plus-v1:0", + "input_price": 0.0000030, + "output_price": 0.0000150 + }, + "ollama": { + "model_name": "ollama", + "input_price": 0, + "output_price": 0 + }, + "claude-3-haiku-20240307": { + "model_name": "claude-3-haiku-20240307", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "claude-3-5-sonnet-20240620": { + "model_name": "claude-3-5-sonnet-20240620", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "claude-3-opus-20240229": { + "model_name": "claude-3-opus-20240229", + "input_price": 0.000015, + "output_price": 0.000075 + }, + "claude-3-sonnet-20240229": { + "model_name": "claude-3-sonnet-20240229", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "command-r-08-202": { + "model_name": "command-r-08-202", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "c4ai-aya-23-8b": { + "model_name": "c4ai-aya-23-8b", + "input_price": 0, + "output_price": 0 + }, + "llama.cpp": { + "model_name": "llama.cpp", + "input_price": 0, + "output_price": 0 + }, + "gpt-4o": { + "model_name": "gpt-4o", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-2024-08-06": { + "model_name": "gpt-4o-2024-08-06", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "gpt-4o-2024-05-13": { + "model_name": "gpt-4o-2024-05-13", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-mini": { + "model_name": "gpt-4o-mini", + "input_price": 0.00000015, + "output_price": 0.0000006 + } +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/values/url.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/values/url.jsonnet new file mode 100644 index 00000000..1bacb067 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/values/url.jsonnet @@ -0,0 +1,6 @@ +{ + pulsar: "pulsar://pulsar:6650", + pulsar_admin: "http://pulsar:8080", + milvus: "http://milvus:19530", + qdrant: "http://qdrant:6333", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/zip-readme.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/zip-readme.md new file mode 100644 index 00000000..0b117792 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.6/zip-readme.md @@ -0,0 +1,28 @@ + +Note! this is a subset of possible configurations, to generate your own +launch config use the config util... + +- Production: https://config-ui.demo.trustgraph.ai +- Early release: https://dev.config-ui.demo.trustgraph.ai + +The config util auto-generates deployment instructions for your +configuration, so that's the recommended way to deploy. + +---------------------------------------------------------------------------- + +These are launch configurations for TrustGraph. See https://trustgraph.ai for +the quickstart using docker compose. + +Hint for Linux: There are files here which get mounted as volumes inside +Docker Compose containers. This may trigger SELinux rules on your system, to +permit access insider the containers, use a command like this... + +chcon -Rt svirt_sandbox_file_t grafana/ prometheus/ + +The file vertexai/private.json is a placeholder for real GCP credentials if +you are using the VertexAI LLM. If you're using that in Docker Compose, +replace with your real credentials, and don't forget to permit access if you +are using Linux: + +chcon -Rt svirt_sandbox_file_t vertexai/ + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/README.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/README.md new file mode 100644 index 00000000..23039e9a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/README.md @@ -0,0 +1,125 @@ + +# TrustGraph template generation + +There are two utilities here: + +- `generate`: Generates a single Docker Compose launch configuration + based on configuration you provide. +- `generate-all`: Generates the release bundle for releases. You won't + need to use this unless you are managing releases. + +## `generate-all` + +Previously, this generates a full set of all vector DB / triple store / LLM +combinations, and put them in a single ZIP file. But this got out of +hand, so at the time of writing, this generates a single configuraton +using Qdrant vector DB, Ollama LLM support and Cassandra for a triple store. + +The combinations are contained withing the code, it takes two arguments: +- output ZIP file (is over-written) +- TrustGraph version number + +``` +templates/generate-all output.zip 0.18.11 +``` + +## `generate` + +This utility takes a configuration file describing the components to bundle, +and outputs a Docker Compose YAML file. + +### Input configuration + +The input configuration is a JSON file, an array of components to pull into +the configuration. For each component, there is a name and a (possibly empty) +object describing addtional parameters for that component. + +Example: + +``` +[ + { + "name": "cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "qdrant", + "parameters": {} + }, + { + "name": "embeddings-hf", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph", + "parameters": {} + }, + { + "name": "googleaistudio", + "parameters": { + "googleaistudio-temperature": 0.3, + "googleaistudio-max-output-tokens": 2048, + "googleaistudio-model": "gemini-1.5-pro-002" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 1000, + "chunk-overlap": 50 + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "agent-manager-react", + "parameters": {} + } +] +``` + +If you want to make your own configuration you could try changing the +configuration above: +- Components which are essential: pulsar, trustgraph, graph-rag, grafana, + agent-manager-react +- You need a triple store, one of: cassandra, memgraph, falkordb, neo4j +- You need a vector store, one of: qdrant, pinecone +- You need an LLM, one of: azure, azure-openai, bedrock, claude, cohere, + llamafile, ollama, openai, vertexai. +- You need an embeddings implementation, one of: embeddings-hf, + embeddings-ollama +- Optionally add the Workbench tool: workbench-ui + +Components have over-ridable parameters, look in the component definition +in `templates/components/` to see what you can override. + +### Invocation + +Two parameters: +- The output ZIP file +- The version number + +The configuration file described above is provided on standard input + +``` +templates/generate out.zip 0.18.9 < config.json +``` + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/base/base.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/base/base.jsonnet new file mode 100644 index 00000000..9f82efb2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/base/base.jsonnet @@ -0,0 +1,3 @@ +{ + restart: "on-failure:100", +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components.jsonnet new file mode 100644 index 00000000..f1c0d176 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components.jsonnet @@ -0,0 +1,99 @@ +{ + + // Essentials + "trustgraph-base": import "components/trustgraph.jsonnet", + "rev-gateway": import "components/rev-gateway.jsonnet", + "pulsar": import "components/pulsar.jsonnet", + + // LLMs + "azure": import "components/azure.jsonnet", + "azure-openai": import "components/azure-openai.jsonnet", + "bedrock": import "components/bedrock.jsonnet", + "claude": import "components/claude.jsonnet", + "cohere": import "components/cohere.jsonnet", + "googleaistudio": import "components/googleaistudio.jsonnet", + "llamafile": import "components/llamafile.jsonnet", + "lmstudio": import "components/lmstudio.jsonnet", + "mistral": import "components/mistral.jsonnet", + "ollama": import "components/ollama.jsonnet", + "openai": import "components/openai.jsonnet", + "vertexai": import "components/vertexai.jsonnet", + "tgi": import "components/tgi.jsonnet", + "vllm": import "components/vllm.jsonnet", + + // LLMs for RAG. RAG components have been collapsed into the core + // component, so gone away. + "azure-rag": {}, + "azure-openai-rag": {}, + "bedrock-rag": {}, + "claude-rag": {}, + "cohere-rag": {}, + "googleaistudio-rag": {}, + "llamafile-rag": {}, + "lmstudio-rag": {}, + "mistral-rag": {}, + "ollama-rag": {}, + "openai-rag": {}, + "vertexai-rag": {}, + "tgi-rag": import "components/tgi-rag.jsonnet", + "vllm-rag": {}, + + "tgi-service-cpu": import "components/tgi-service-cpu.jsonnet", + "tgi-service-intel-gpu": import "components/tgi-service-intel-gpu.jsonnet", + "tgi-service-gaudi": import "components/tgi-service-gaudi.jsonnet", + + "vllm-service-intel-gpu": import "components/vllm-service-intel-gpu.jsonnet", + "vllm-service-gaudi": import "components/vllm-service-gaudi.jsonnet", + "vllm-service-nvidia": import "components/vllm-service-nvidia.jsonnet", + + // Embeddings + "embeddings-ollama": import "components/embeddings-ollama.jsonnet", + "embeddings-hf": import "components/embeddings-hf.jsonnet", + "embeddings-fastembed": import "components/embeddings-fastembed.jsonnet", + + // OCR options + "ocr": import "components/ocr.jsonnet", + "mistral-ocr": import "components/mistral-ocr.jsonnet", + + // Vector stores + "vector-store-milvus": import "components/vector-store-milvus.jsonnet", + "vector-store-qdrant": import "components/vector-store-qdrant.jsonnet", + "vector-store-pinecone": import "components/vector-store-pinecone.jsonnet", + + // Triples stores + "triple-store-cassandra": import "components/triple-store-cassandra.jsonnet", + "triple-store-neo4j": import "components/triple-store-neo4j.jsonnet", + "triple-store-falkordb": import "components/triple-store-falkordb.jsonnet", + "triple-store-memgraph": import "components/triple-store-memgraph.jsonnet", + + // Object stores + "row-store-cassandra": import "components/row-store-cassandra.jsonnet", + + // Observability support + "grafana": import "components/grafana.jsonnet", + "loki": import "components/loki.jsonnet", + + // Pulsar manager is a UI for Pulsar. Uses a LOT of memory + "pulsar-manager": import "components/pulsar-manager.jsonnet", + + "override-recursive-chunker": import "components/chunker-recursive.jsonnet", + + // The prompt manager + "prompt-overrides": import "components/prompt-overrides.jsonnet", + + // Archaic - part of core system, just making sure these don't + // cause a failure + "workbench-ui": {}, + "prompt-template": {}, + "agent-manager-react": {}, + "graph-rag": {}, + "document-rag": {}, + "librarian": {}, + + // Extra MCP services + "ddg-mcp-server": import "mcp/ddg-mcp-server.jsonnet", + + // Does nothing. But, can be a hack to overwrite parameters + "null": {}, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/agent-manager-react.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/agent-manager-react.jsonnet new file mode 100644 index 00000000..d99ae88c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/agent-manager-react.jsonnet @@ -0,0 +1,40 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "agent-manager" +: { + + create:: function(engine) + + local container = + engine.container("agent-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "agent-manager-react", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "agent-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/azure-openai.jsonnet new file mode 100644 index 00000000..4d5a70e8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/azure-openai.jsonnet @@ -0,0 +1,113 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure-openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-openai-" + key]:: value, + }, + +// Strategy is to specify the model with the AZURE_MODEL environment +// variable. This isn't something that can just be specified dynamically, +// it has to match what was provisioned in Azure. + + "azure-openai-max-output-tokens":: 4192, + "azure-openai-temperature":: 0.0, + "azure-openai-models":: models, + + "llm-models" +:: $["azure-openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/azure.jsonnet new file mode 100644 index 00000000..f4db7500 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/azure.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-" + key]:: value, + }, + + "azure-max-output-tokens":: 4096, + "azure-temperature":: 0.0, + "azure-models":: models, + + "llm-models" +:: $["azure-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/bedrock.jsonnet new file mode 100644 index 00000000..19bf1459 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/bedrock.jsonnet @@ -0,0 +1,106 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local chunker = import "chunker-recursive.jsonnet"; +local models = import "parameters/bedrock.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["bedrock-" + key]:: value, + }, + + "bedrock-max-output-tokens":: 4096, + "bedrock-temperature":: 0.0, + "bedrock-models":: models, + + "llm-models" +:: $["bedrock-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + chunker + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/chunker-recursive.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/chunker-recursive.jsonnet new file mode 100644 index 00000000..a9f31d73 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/chunker-recursive.jsonnet @@ -0,0 +1,48 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "chunk-size":: 2000, + "chunk-overlap":: 100, + + "chunker" +: { + + create:: function(engine) + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + std.toString($["chunk-size"]), + "--chunk-overlap", + std.toString($["chunk-overlap"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/claude.jsonnet new file mode 100644 index 00000000..f2dcc7e1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/claude.jsonnet @@ -0,0 +1,105 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/claude.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["claude-" + key]:: value, + }, + + "claude-max-output-tokens":: 4096, + "claude-temperature":: 0.0, + "claude-models":: models, + + "llm-models" +:: $["claude-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/cohere.jsonnet new file mode 100644 index 00000000..644b7b4a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/cohere.jsonnet @@ -0,0 +1,98 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/cohere.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["cohere-" + key]:: value, + }, + + "cohere-temperature":: 0.0, + "cohere-models":: models, + + "llm-models" +:: $["cohere-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/configuration.jsonnet new file mode 100644 index 00000000..b2499ff6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/configuration.jsonnet @@ -0,0 +1,50 @@ + +// This puts the default configuration together. References many things, +// flow classes, a default flow, token costs, prompts, agent tools + +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "init-trustgraph" +: { + + create:: function(engine) + + local cfgVol = engine.configVolume( + "trustgraph-cfg", "trustgraph", + { + "config.json": importstr "trustgraph/config.json", + } + ); + + local container = + engine.container("init-trustgraph") + .with_image(images.trustgraph_flow) + .with_command( + [ + "tg-init-trustgraph", + "-p", + url.pulsar_admin, + "--config-file", + "/trustgraph/config.json", + ] + ) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M") + .with_volume_mount(cfgVol, "/trustgraph/"); + + local containerSet = engine.containers( + "init-trustgraph", [ container ] + ); + + engine.resources([ + cfgVol, + containerSet, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/document-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/document-rag.jsonnet new file mode 100644 index 00000000..239598b7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/document-rag.jsonnet @@ -0,0 +1,77 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "document-rag-doc-limit":: 20, + + "document-rag" +: { + + create:: function(engine) + + local container = + engine.container("document-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "document-rag", + "-p", + url.pulsar, + "--doc-limit", + std.toString($["document-rag-doc-limit"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "document-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "document-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("document-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "document-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M"); + + local containerSet = engine.containers( + "document-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..1e4d6cb9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/embeddings-fastembed.jsonnet @@ -0,0 +1,48 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-fastembed.jsonnet"; + +{ + + "fastembed-models":: models, + + "embeddings-models" +:: $["fastembed-models"], + + embeddings +: { + + create:: function(engine) + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-fastembed", + "-p", + url.pulsar, + "--concurrency", + std.toString($["embeddings-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/embeddings-hf.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/embeddings-hf.jsonnet new file mode 100644 index 00000000..29008129 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/embeddings-hf.jsonnet @@ -0,0 +1,46 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-huggingface.jsonnet"; + +{ + + "huggingface-embeddings-models":: models, + + "embeddings-models" +:: $["huggingface-embeddings-models"], + + embeddings +: { + + create:: function(engine) + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_hf) + .with_command([ + "embeddings-hf", + "-p", + url.pulsar, + "--concurrency", + std.toString($["embeddings-concurrency"]), + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/embeddings-ollama.jsonnet new file mode 100644 index 00000000..8422b646 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/embeddings-ollama.jsonnet @@ -0,0 +1,51 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local models = import "parameters/embeddings-ollama.jsonnet"; + +{ + + "ollama-url":: "${OLLAMA_HOST}", + + "ollama-models":: models, + + "embeddings-models" +:: $["ollama-models"], + + embeddings +: { + + create:: function(engine) + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString($["embeddings-concurrency"]), + "-r", + $["ollama-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/googleaistudio.jsonnet new file mode 100644 index 00000000..9b6ff6c5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/googleaistudio.jsonnet @@ -0,0 +1,105 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/googleaistudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["googleaistudio-" + key]:: value, + }, + + "googleaistudio-max-output-tokens":: 4096, + "googleaistudio-temperature":: 0.0, + "googleaistudio-models":: models, + + "llm-models" +:: $["googleaistudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/grafana.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/grafana.jsonnet new file mode 100644 index 00000000..decdf4f3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/grafana.jsonnet @@ -0,0 +1,125 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local loki = import "loki.jsonnet"; + +{ + + "prometheus" +: { + + create:: function(engine) + + local vol = engine.volume("prometheus-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "prometheus-cfg", "prometheus", + { + "prometheus.yml": importstr "prometheus/prometheus.yml", + } + ); + + local container = + engine.container("prometheus") + .with_image(images.prometheus) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M") + .with_port(9090, 9090, "http") + .with_volume_mount(cfgVol, "/etc/prometheus/") + .with_volume_mount(vol, "/prometheus"); + + local containerSet = engine.containers( + "prometheus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9090, 9090, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + + "grafana" +: { + + create:: function(engine) + + local vol = engine.volume("grafana-storage").with_size("20G"); + + local provDashVol = engine.configVolume( + "prov-dash", "grafana/provisioning/", + { + "dashboard.yml": + importstr "grafana/provisioning/dashboard.yml", + } + + ); + + local provDataVol = engine.configVolume( + "prov-data", "grafana/provisioning/", + { + "datasource.yml": + importstr "grafana/provisioning/datasource.yml", + } + + ); + + local dashVol = engine.configVolume( + "dashboards", "grafana/dashboards/", + { + "overview-dashboard.json": + importstr "grafana/dashboards/overview-dashboard.json", + "log-dashboard.json": + importstr "grafana/dashboards/log-dashboard.json", + } + + ); + + local container = + engine.container("grafana") + .with_image(images.grafana) + .with_environment({ + // GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin", + // GF_AUTH_ANONYMOUS_ENABLED: "true", + // GF_ORG_ROLE: "Admin", + GF_ORG_NAME: "trustgraph.ai", + // GF_SERVER_ROOT_URL: "https://example.com", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(3000, 3000, "cassandra") + .with_volume_mount(vol, "/var/lib/grafana") + .with_volume_mount( + provDashVol, "/etc/grafana/provisioning/dashboards/" + ) + .with_volume_mount( + provDataVol, "/etc/grafana/provisioning/datasources/" + ) + .with_volume_mount( + dashVol, "/var/lib/grafana/dashboards/" + ); + + local containerSet = engine.containers( + "grafana", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3000, 3000, "http"); + + engine.resources([ + vol, + provDashVol, + provDataVol, + dashVol, + containerSet, + service, + ]) + + }, + +} + loki + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/graph-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/graph-rag.jsonnet new file mode 100644 index 00000000..64ab0130 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/graph-rag.jsonnet @@ -0,0 +1,223 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "graph-rag-entity-limit":: 50, + "graph-rag-triple-limit":: 30, + "graph-rag-max-subgraph-size":: 400, + "graph-rag-max-path-length":: 2, + + "kg-extract-definitions" +: { + + create:: function(engine) + + local container = + engine.container("kg-extract-definitions") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-definitions", + "-p", + url.pulsar, + "--concurrency", + std.toString($["kg-extraction-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-extract-definitions", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-relationships" +: { + + create:: function(engine) + + local container = + engine.container("kg-extract-relationships") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-relationships", + "-p", + url.pulsar, + "--concurrency", + std.toString($["kg-extraction-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-extract-relationships", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-agent" +: { + + create:: function(engine) + + local container = + engine.container("kg-extract-agent") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-agent", + "-p", + url.pulsar, + "--concurrency", + std.toString($["kg-extraction-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-extract-agent", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-ontology" +: { + + create:: function(engine) + + local container = + engine.container("kg-extract-ontology") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-ontology", + "-p", + url.pulsar, + "--concurrency", + std.toString($["kg-extraction-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "300M") + .with_reservations("0.1", "300M"); + + local containerSet = engine.containers( + "kg-extract-ontology", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-rag" +: { + + create:: function(engine) + + local container = + engine.container("graph-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-rag", + "-p", + url.pulsar, +// "--concurrency", +// std.toString($["graph-rag-concurrency"]), + "--entity-limit", + std.toString($["graph-rag-entity-limit"]), + "--triple-limit", + std.toString($["graph-rag-triple-limit"]), + "--max-subgraph-size", + std.toString($["graph-rag-max-subgraph-size"]), + "--max-path-length", + std.toString($["graph-rag-max-path-length"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "graph-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M"); + + local containerSet = engine.containers( + "graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/librarian.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/librarian.jsonnet new file mode 100644 index 00000000..cfcbcff0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/librarian.jsonnet @@ -0,0 +1,43 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local minio = import "stores/minio.jsonnet"; +local cassandra = import "stores/cassandra.jsonnet"; + +{ + + "librarian" +: { + + create:: function(engine) + + local container = + engine.container("librarian") + .with_image(images.trustgraph_flow) + .with_command([ + "librarian", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M"); + + local containerSet = engine.containers( + "librarian", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +// Minio and Cassandra are used by the Librarian +} + minio + cassandra + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/llamafile.jsonnet new file mode 100644 index 00000000..bb6c55fd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/llamafile.jsonnet @@ -0,0 +1,95 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/slm.jsonnet"; +local models = import "parameters/llamafile.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["llamafile-" + key]:: value, + }, + + "llamafile-models":: models, + + "llm-models" +:: $["llamafile-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/lmstudio.jsonnet new file mode 100644 index 00000000..a578b694 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/lmstudio.jsonnet @@ -0,0 +1,105 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/lmstudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["lmstudio-" + key]:: value, + }, + + "lmstudio-max-output-tokens":: 4096, + "lmstudio-temperature":: 0.0, + "lmstudio-models":: models, + + "llm-models" +:: $["lmstudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/loki.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/loki.jsonnet new file mode 100644 index 00000000..35bbff3b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/loki.jsonnet @@ -0,0 +1,46 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "loki" +: { + + create:: function(engine) + + local vol = engine.volume("loki-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "loki-cfg", "loki", + { + "local-config.yaml": importstr "loki/local-config.yaml", + } + ); + + local container = + engine.container("loki") + .with_image(images.loki) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M") + .with_port(3100, 3100, "http") + .with_volume_mount(cfgVol, "/etc/loki/") + .with_volume_mount(vol, "/loki"); + + local containerSet = engine.containers( + "loki", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3100, 3100, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/mcp-server.jsonnet new file mode 100644 index 00000000..7769889e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/mcp-server.jsonnet @@ -0,0 +1,48 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "mcp-server-port":: 8000, + + "mcp-server" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mcp-server-secret") + .with_env_var("MCP_SERVER_SECRET", "mcp-server-secret"); + + local port = $["mcp-server-port"]; + + local container = + engine.container("mcp-server") + .with_image(images.trustgraph_mcp) + .with_command([ + "mcp-server", + "--port", + std.toString(port), + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/mistral-ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/mistral-ocr.jsonnet new file mode 100644 index 00000000..f70e0cd6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/mistral-ocr.jsonnet @@ -0,0 +1,49 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "pdf-decoder" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("mistral-ocr") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-ocr-mistral", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "mistral-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/mistral.jsonnet new file mode 100644 index 00000000..6d01fbc9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/mistral.jsonnet @@ -0,0 +1,105 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/mistral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "mistral-max-output-tokens":: 4096, + "mistral-temperature":: 0.0, + "mistral-models":: models, + + "llm-models" +:: $["mistral-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/null.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/null.jsonnet new file mode 100644 index 00000000..2c63c085 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/null.jsonnet @@ -0,0 +1,2 @@ +{ +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/object-store-cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/object-store-cassandra.jsonnet new file mode 100644 index 00000000..8362b5c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/object-store-cassandra.jsonnet @@ -0,0 +1,78 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "stores/cassandra.jsonnet"; + +cassandra + { + + "store-objects" +: { + + create:: function(engine) + + local container = + engine.container("store-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "objects-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-objects" +: { + + create:: function(engine) + + local container = + engine.container("query-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "objects-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/ocr.jsonnet new file mode 100644 index 00000000..4f53aa5a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/ocr.jsonnet @@ -0,0 +1,37 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "pdf-decoder" +: { + + create:: function(engine) + + local container = + engine.container("pdf-ocr") + .with_image(images.trustgraph_ocr) + .with_command([ + "pdf-ocr", + "-p", + url.pulsar, + ]) + .with_limits("1.0", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "pdf-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/ollama.jsonnet new file mode 100644 index 00000000..bccfd0f1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/ollama.jsonnet @@ -0,0 +1,99 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/ollama.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["ollama-" + key]:: value, + }, + + "ollama-models":: models, + + "llm-models" +:: $["ollama-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString($["text-completion-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString($["text-completion-rag-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/openai.jsonnet new file mode 100644 index 00000000..c8754280 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/openai.jsonnet @@ -0,0 +1,107 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["openai-" + key]:: value, + }, + + "openai-max-output-tokens":: 4096, + "openai-temperature":: 0.0, + "openai-models":: models, + + "llm-models" +:: $["openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/prompt-overrides.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/prompt-overrides.jsonnet new file mode 100644 index 00000000..852ec09d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/prompt-overrides.jsonnet @@ -0,0 +1,24 @@ +local default_prompts = import "prompts/default-prompts.jsonnet"; + +{ + + with:: function(key, value) + if (key == "system-template") then + self + { + prompts +:: { + "system-template": value, + } + } + else + self + { + prompts +:: { + templates +:: { + [key] +:: { + prompt: value + } + } + } + }, + +} + default_prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/prompt-template.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/prompt-template.jsonnet new file mode 100644 index 00000000..5d1af35b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/prompt-template.jsonnet @@ -0,0 +1,78 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "prompt" +: { + + create:: function(engine) + + local container = + engine.container("prompt") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--concurrency", + std.toString($["prompt-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "prompt", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "prompt-rag" +: { + + create:: function(engine) + + local container = + engine.container("prompt-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--id", + "prompt-rag", + "--concurrency", + std.toString($["prompt-rag-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "prompt-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/pulsar-manager.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/pulsar-manager.jsonnet new file mode 100644 index 00000000..9a0b59b2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/pulsar-manager.jsonnet @@ -0,0 +1,41 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "pulsar" +: { + + create:: function(engine) + +// FIXME: Should persist something? +// local volume = engine.volume(...) + + local container = + engine.container("pulsar") + .with_image(images.pulsar_manager) + .with_environment({ + SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties", + }) + .with_limits("0.5", "1.4G") + .with_reservations("0.1", "1.4G") + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2"); + + local containerSet = engine.containers( + "pulsar", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2); + + engine.resources([ + containerSet, + service, + ]) + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/pulsar.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/pulsar.jsonnet new file mode 100644 index 00000000..0cfd1854 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/pulsar.jsonnet @@ -0,0 +1,171 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +// This is a Pulsar configuration. Non-standalone mode so we deploy +// individual components: bookkeeper, broker and zookeeper. +// +// This also deploys the TrustGraph 'admin' container which initialises +// TrustGraph-specific namespaces etc. + +{ + + "pulsar" +: { + + create:: function(engine) + + // Zookeeper volume + local zkVolume = engine.volume("zookeeper").with_size("1G"); + + // Zookeeper container + local zkContainer = + engine.container("zookeeper") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper" + ]) + .with_limits("1", "400M") + .with_reservations("0.05", "400M") + .with_user("0:1000") + .with_volume_mount(zkVolume, "/pulsar/data/zookeeper") + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m", + }) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Pulsar cluster init container + local initContainer = + engine.container("pulsar-init") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650", + ]) + .with_limits("1", "512M") + .with_reservations("0.05", "512M") + .with_environment({ + "PULSAR_MEM": "-Xms256m -Xmx256m -XX:MaxDirectMemorySize=256m", + }); + + + // Bookkeeper volume + local bookieVolume = engine.volume("bookie").with_size("20G"); + + // Bookkeeper container + local bookieContainer = + engine.container("bookie") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + // false ^ causes this to be a 'failure' exit. + ]) + .with_limits("1", "1024M") + .with_reservations("0.1", "1024M") + .with_user("0:1000") + .with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper") + .with_environment({ + "clusterName": "cluster-a", + "zkServers": "zookeeper:2181", + "bookieId": "bookie", + "metadataStoreUri": "metadata-store:zk:zookeeper:2181", + "advertisedAddress": "bookie", + "BOOKIE_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m", + }) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker, stateless (uses ZK and Bookkeeper for state) + local brokerContainer = + engine.container("pulsar") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" + ]) + .with_limits("1", "800M") + .with_reservations("0.1", "800M") + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "zookeeperServers": "zookeeper:2181", + "clusterName": "cluster-a", + "managedLedgerDefaultEnsembleSize": "1", + "managedLedgerDefaultWriteQuorum": "1", + "managedLedgerDefaultAckQuorum": "1", + "advertisedAddress": "pulsar", + "advertisedListeners": "external:pulsar://pulsar:6650,localhost:pulsar://localhost:6650", + "PULSAR_MEM": "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m", + }) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + // Container sets + local zkContainerSet = engine.containers( + "zookeeper", + [ + zkContainer, + ] + ); + + local initContainerSet = engine.containers( + "init-pulsar", + [ + initContainer, + ] + ); + + local bookieContainerSet = engine.containers( + "bookie", + [ + bookieContainer, + ] + ); + + local brokerContainerSet = engine.containers( + "pulsar", + [ + brokerContainer, + ] + ); + + // Zookeeper service + local zkService = + engine.service(zkContainerSet) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Bookkeeper service + local bookieService = + engine.service(bookieContainerSet) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker service + local brokerService = + engine.service(brokerContainerSet) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + engine.resources([ + zkVolume, + bookieVolume, + zkContainerSet, + initContainerSet, + bookieContainerSet, + brokerContainerSet, + zkService, + bookieService, + brokerService, + ]) + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/rev-gateway.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/rev-gateway.jsonnet new file mode 100644 index 00000000..b549f402 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/rev-gateway.jsonnet @@ -0,0 +1,54 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + // Invalid, but at least means the rev-gateway won't connect to anything + // it shouldn't. + "rev-gateway-token":: "INVALID_TOKEN", + "rev-gateway-uri":: "wss://127.0.0.1/api/v1/relay?token=" + $["rev-gateway-token"], + + "rev-gateway" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("rev-gateway-secret") + .with_env_var("REV_GATEWAY_SECRET", "rev-gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "rev-gateway", + "-p", + url.pulsar, + "--websocket-uri", + std.toString($["rev-gateway-uri"]), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/structured-data.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/structured-data.jsonnet new file mode 100644 index 00000000..33d0ef06 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/structured-data.jsonnet @@ -0,0 +1,137 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "nlp-query" +: { + + create:: function(engine) + + local container = + engine.container("nlp-query") + .with_image(images.trustgraph_flow) + .with_command([ + "nlp-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "nlp-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-query" +: { + + create:: function(engine) + + local container = + engine.container("structured-query") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "structured-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-diag" +: { + + create:: function(engine) + + local container = + engine.container("structured-diag") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-diag", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "96M") + .with_reservations("0.1", "96M"); + + local containerSet = engine.containers( + "structured-diag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-objects" +: { + + create:: function(engine) + + local container = + engine.container("kg-extract-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-objects", + "-p", + url.pulsar, + "--concurrency", + std.toString($["kg-extraction-concurrency"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-extract-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-rag.jsonnet new file mode 100644 index 00000000..0a8a7af9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-rag.jsonnet @@ -0,0 +1,62 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-rag-" + key]:: value, + }, + + "tgi-rag-max-output-tokens":: 1024, + "tgi-rag-temperature":: 0.0, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString($["text-completion-rag-concurrency"]), + "-x", + std.toString($["tgi-rag-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-rag-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-service-cpu.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-service-cpu.jsonnet new file mode 100644 index 00000000..0a02f210 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-service-cpu.jsonnet @@ -0,0 +1,65 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage").with_size("20G"); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-cpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--cuda-graphs", + "0", + "--port", + "8899" + ]) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_environment({ + HF_TOKEN: $["hf-token"], + }) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(8899, 8899, "tgi") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-service-gaudi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-service-gaudi.jsonnet new file mode 100644 index 00000000..9c52ef3d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-service-gaudi.jsonnet @@ -0,0 +1,92 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "meta-llama/Llama-3.3-70B-Instruct", + "tgi-service-cpus":: "64.0", + "tgi-service-memory":: "64G", + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage").with_size("50G"); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-gaudi"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--sharded", + "true", + "--num-shard", + "8", + "--max-input-tokens", + "4096", + "--max-total-tokens", + "4096", + "--max-batch-size", + "128", +// "--max-batch-prefill-tokens", +// "16384", + "--max-waiting-tokens", + "7", +// "--waiting-served-ratio", +// "1.2", + "--max-concurrent-requests", + "512", + "--cuda-graphs", + "0", + "--port", + "8899" + ]) + .with_runtime("habana") + .with_environment({ + HABANA_VISIBLE_DEVICES: "all", + OMPI_MCA_btl_vader_single_copy_mechanism: "none", + HF_TOKEN: $["hf-token"], + ENABLE_HPU_GRAPH: 'true', + LIMIT_HPU_GRAPH: 'true', + USE_FLASH_ATTENTION: 'true', + FLASH_ATTENTION_RECOMPUTE: 'true', +// PT_HPU_ENABLE_LAZY_COLLECTIVES: 'true', +// PREFILL_BATCH_BUCKET_SIZE: "1", +// BATCH_BUCKET_SIZE: "1", + + }) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(8899, 8899, "tgi") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-service-intel-gpu.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-service-intel-gpu.jsonnet new file mode 100644 index 00000000..ef223288 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi-service-intel-gpu.jsonnet @@ -0,0 +1,65 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage").with_size("20G"); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-intel-xpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--cuda-graphs", + "0", + "--port", + "8899" + ]) + .with_environment({ + HF_TOKEN: $["hf-token"], + }) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(8899, 8899, "tgi") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi.jsonnet new file mode 100644 index 00000000..f62266a2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/tgi.jsonnet @@ -0,0 +1,60 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-" + key]:: value, + }, + + "tgi-max-output-tokens":: 1024, + "tgi-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--concurrency", + std.toString($["text-completion-concurrency"]), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-cassandra.jsonnet new file mode 100644 index 00000000..a7106cb5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-cassandra.jsonnet @@ -0,0 +1,78 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "stores/cassandra.jsonnet"; + +cassandra + { + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-falkordb.jsonnet new file mode 100644 index 00000000..f9177a18 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-falkordb.jsonnet @@ -0,0 +1,80 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local falkordb = import "stores/falkordb.jsonnet"; + +falkordb + { + + "falkordb-url":: "falkor://falkordb:6379", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-memgraph.jsonnet new file mode 100644 index 00000000..bc86a5a7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-memgraph.jsonnet @@ -0,0 +1,85 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local memgraph = import "stores/memgraph.jsonnet"; + +memgraph + { + + "memgraph-url":: "bolt://memgraph:7687", + "memgraph-database":: "memgraph", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-neo4j.jsonnet new file mode 100644 index 00000000..f4a93d9c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/triple-store-neo4j.jsonnet @@ -0,0 +1,80 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local neo4j = import "stores/neo4j.jsonnet"; + +neo4j + { + + "neo4j-url":: "bolt://neo4j:7687", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/trustgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/trustgraph.jsonnet new file mode 100644 index 00000000..1eac5dd8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/trustgraph.jsonnet @@ -0,0 +1,349 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +local config_initialiser = import "configuration.jsonnet"; +local config = import "trustgraph-config.jsonnet"; +local librarian = import "librarian.jsonnet"; +local mcp_server = import "mcp-server.jsonnet"; +local workbench = import "workbench-ui.jsonnet"; +local graphrag = import "graph-rag.jsonnet"; +local documentrag = import "document-rag.jsonnet"; +local prompt_template = import "prompt-template.jsonnet"; +local agent_manager = import "agent-manager-react.jsonnet"; +local structured_data = import "structured-data.jsonnet"; +local ddg = import "mcp/ddg-mcp-server.jsonnet"; + +{ + + "log-level":: "INFO", + + "api-gateway-port":: 8088, + "api-gateway-timeout":: 600, + + "chunk-size":: 250, + "chunk-overlap":: 15, + + "prompt-concurrency":: 1, + "prompt-rag-concurrency":: 1, + + "text-completion-concurrency":: 1, + "text-completion-rag-concurrency":: 1, + + "kg-extraction-concurrency":: 1, + "graph-rag-concurrency":: 1, + + "embeddings-concurrency":: 1, + + "api-gateway" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("gateway-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local port = $["api-gateway-port"]; + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "api-gateway", + "-p", + url.pulsar, + "--timeout", + std.toString($["api-gateway-timeout"]), + "--port", + std.toString(port), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "chunker" +: { + + create:: function(engine) + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-token", + "-p", + url.pulsar, + "--chunk-size", + std.toString($["chunk-size"]), + "--chunk-overlap", + std.toString($["chunk-overlap"]), + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "config-svc" +: { + + create:: function(engine) + + local container = + engine.container("config-svc") + .with_image(images.trustgraph_flow) + .with_command([ + "config-svc", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "config-svc", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "pdf-decoder" +: { + + create:: function(engine) + + local container = + engine.container("pdf-decoder") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-decoder", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "pdf-decoder", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "mcp-tool" +: { + + create:: function(engine) + + local container = + engine.container("mcp-tool") + .with_image(images.trustgraph_flow) + .with_command([ + "mcp-tool", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "mcp-tool", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering" +: { + + create:: function(engine) + + local container = + engine.container("metering") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "metering", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering-rag" +: { + + create:: function(engine) + + local container = + engine.container("metering-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--id", + "metering-rag", + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "metering-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-store" +: { + + create:: function(engine) + + local container = + engine.container("kg-store") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-store", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-store", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-manager" +: { + + create:: function(engine) + + local container = + engine.container("kg-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-manager", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "kg-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + librarian + mcp_server + workbench + graphrag + + documentrag + prompt_template + agent_manager + structured_data + + config_initialiser + config + + ddg + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vector-store-milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vector-store-milvus.jsonnet new file mode 100644 index 00000000..32d6fab4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vector-store-milvus.jsonnet @@ -0,0 +1,147 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local milvus = import "stores/milvus.jsonnet"; + +milvus + { + + "store-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vector-store-pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vector-store-pinecone.jsonnet new file mode 100644 index 00000000..0f8d6a33 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vector-store-pinecone.jsonnet @@ -0,0 +1,161 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; + +{ + + "pinecone-cloud":: "aws", + "pinecone-region":: "us-east-1", + + "store-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vector-store-qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vector-store-qdrant.jsonnet new file mode 100644 index 00000000..0582cd2f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vector-store-qdrant.jsonnet @@ -0,0 +1,147 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local qdrant = import "stores/qdrant.jsonnet"; + +qdrant + { + + "store-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vertexai.jsonnet new file mode 100644 index 00000000..05f608ce --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vertexai.jsonnet @@ -0,0 +1,121 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vertexai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vertexai-" + key]:: value, + }, + + "vertexai-private-key":: "/vertexai/private.json", + "vertexai-region":: "us-central1", + "vertexai-max-output-tokens":: 4096, + "vertexai-temperature":: 0.0, + "vertexai-models":: models, + + "llm-models" +:: $["vertexai-models"], + + "text-completion" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion-rag") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm-service-gaudi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm-service-gaudi.jsonnet new file mode 100644 index 00000000..25c1d921 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm-service-gaudi.jsonnet @@ -0,0 +1,66 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "64.0", + "vllm-service-memory":: "64G", + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage").with_size("50G"); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-gaudi"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--tensor-parallel-size=8", + "--port", + "8899", + ]) + .with_runtime("habana") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + HUGGING_FACE_HUB_TOKEN: $["hf-token"], + HABANA_VISIBLE_DEVICES: "all", + VLLM_CACHE_ROOT: "/data", + }) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(8899, 8899, "vllm") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm-service-intel-gpu.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm-service-intel-gpu.jsonnet new file mode 100644 index 00000000..cdd98abc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm-service-intel-gpu.jsonnet @@ -0,0 +1,77 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "8.0", + "vllm-service-memory":: "16G", + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage").with_size("20G"); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-xpu"]) + .with_command([ + "python", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--dtype=float16", + "--device=xpu", + "--enforce-eager", + "--port", + "8899", + "--block-size", + "64", + "--gpu-memory-util", + "0.85", + "--trust-remote-code", + "--disable-sliding-window", + ]) + .with_environment({ + HF_TOKEN: $["hf-token"], + VLLM_USE_V1: "1", + W_LONG_MAX_MODEL_LEN: "1", + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + }) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(8899, 8899, "vllm") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm-service-nvidia.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm-service-nvidia.jsonnet new file mode 100644 index 00000000..42f23530 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm-service-nvidia.jsonnet @@ -0,0 +1,64 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-7B-Instruct-v0.3", + "vllm-service-cpus":: "0.5", + "vllm-service-memory":: "1G", + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage").with_size("50G"); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-nvidia"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--port", + "8899", + ]) + .with_runtime("nvidia") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + HUGGING_FACE_HUB_TOKEN: $["hf-token"], + VLLM_CACHE_ROOT: "/data", + }) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(8899, 8899, "vllm") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(8899, 8899, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm.jsonnet new file mode 100644 index 00000000..63dfe84b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/vllm.jsonnet @@ -0,0 +1,110 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vllm.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-" + key]:: value, + }, + + "vllm-models":: models, + + "llm-models" +:: $["vllm-models"], + + "vllm-max-output-tokens":: 1024, + "vllm-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--concurrency", + std.toString($["text-completion-concurrency"]), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString($["text-completion-rag-concurrency"]), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/workbench-ui.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/workbench-ui.jsonnet new file mode 100644 index 00000000..f2048e47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/components/workbench-ui.jsonnet @@ -0,0 +1,32 @@ +local images = import "values/images.jsonnet"; + +{ + + "workbench-ui" +: { + + create:: function(engine) + + local container = + engine.container("workbench-ui") + .with_image(images["workbench-ui"]) + .with_limits("0.1", "256M") + .with_reservations("0.1", "256M") + .with_port(8888, 8888, "ui"); + + local containerSet = engine.containers( + "workbench-ui", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8888, 8888, "ui"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-aks-k8s.jsonnet new file mode 100644 index 00000000..c603a0d8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-aks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "engine/aks-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-docker-compose.jsonnet new file mode 100644 index 00000000..442d2cb7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-docker-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "engine/docker-compose.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-eks-k8s.jsonnet new file mode 100644 index 00000000..af8f8ed8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-eks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "engine/eks-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-gcp-k8s.jsonnet new file mode 100644 index 00000000..3d089a24 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-gcp-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "engine/gcp-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-minikube-k8s.jsonnet new file mode 100644 index 00000000..073358cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-minikube-k8s.jsonnet @@ -0,0 +1,26 @@ + +local engine = import "engine/minikube-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +// Extract resources using the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-noop.jsonnet new file mode 100644 index 00000000..cae548f7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-noop.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "engine/noop.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-ovh-k8s.jsonnet new file mode 100644 index 00000000..9c18f449 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-ovh-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "engine/ovh-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-podman-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-podman-compose.jsonnet new file mode 100644 index 00000000..442d2cb7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-podman-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "engine/docker-compose.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-scw-k8s.jsonnet new file mode 100644 index 00000000..98b79254 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-scw-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "engine/scw-k8s.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-tg-configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-tg-configuration.jsonnet new file mode 100644 index 00000000..4d4c6711 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config-to-tg-configuration.jsonnet @@ -0,0 +1,15 @@ + +local engine = import "engine/noop.jsonnet"; +local decode = import "util/decode-config.jsonnet"; +local components = import "components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract configuration directly from patterns +patterns.configuration.configuration + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/config-composer.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/config-composer.jsonnet new file mode 100644 index 00000000..52967736 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/config-composer.jsonnet @@ -0,0 +1,97 @@ +// Configuration Composer Module +// Orchestrates the complete configuration building process +// Combines all components into the final TrustGraph configuration + +local flow_builder = import "flow-builder.jsonnet"; +local interface_builder = import "interface-builder.jsonnet"; + +{ + // Main function to build the complete configuration + build: function(config_spec) + // Extract configuration parameters + local flow_classes = config_spec.flow_classes; + local default_flow_class = config_spec.default_flow_class; + local default_flow_id = config_spec.default_flow_id; + local flow_init_parameters = config_spec.flow_init_parameters; + + // Build all processors for the default flow + local class_processors = flow_builder.build_class_processors( + flow_classes, + default_flow_class, + flow_init_parameters + ); + + local flow_processors = flow_builder.build_flow_processors( + flow_classes, + default_flow_class, + default_flow_id, + flow_init_parameters + ); + + // Combine processors into flow objects + local processor_array = class_processors + flow_processors; + local flow_objects = flow_builder.build_flow_objects(processor_array); + local flows_active = flow_builder.merge_flow_objects(flow_objects); + + // Build interfaces for the default flow + local default_flow_interfaces = interface_builder.build_interfaces( + flow_classes, + default_flow_class, + default_flow_id, + flow_init_parameters + ); + + // Return object with nested configuration (for backwards compatibility) + { + // Create function (for backwards compatibility) + create: function(engine) {}, + + // The actual configuration object + configuration: { + // Prompts configuration + prompt: { + "system": config_spec.prompts["system-template"], + "template-index": std.objectFieldsAll(config_spec.prompts.templates), + } + { + ["template." + template.key]: template.value + for template in std.objectKeysValuesAll(config_spec.prompts.templates) + }, + + // Tools configuration + tool: { + [tool.id]: tool + for tool in config_spec.tools + }, + + // MCP configuration + mcp: config_spec.mcp, + + // Flow classes reference + "flow-classes": flow_classes, + + // Interface descriptions + "interface-descriptions": config_spec.interface_descriptions, + + // Flow instances + "flows": { + [default_flow_id]: { + "description": "Default processing flow", + "class-name": default_flow_class, + "interfaces": default_flow_interfaces, + "parameters": flow_init_parameters, + }, + }, + + // Active flow processors + "flows-active": flows_active, + + // Token costs and parameter types + "token-costs": config_spec.token_costs, + "parameter-types": config_spec.parameter_types, + + // Collections configuration + "collection": config_spec.collection, + + }, + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/flow-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/flow-builder.jsonnet new file mode 100644 index 00000000..eff7c893 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/flow-builder.jsonnet @@ -0,0 +1,72 @@ +// Flow Builder Module +// Processes flow classes and builds complete flow configurations +// Handles {class}, {id}, and parameter substitutions + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds class-level processors with parameter substitution + // Processes the 'class' section of flow classes + build_class_processors: function(flow_classes, class_name, parameters) + [ + [ + // Replace {class} in the processor key + local key = std.strReplace(processor.key, "{class}", class_name); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // First replace {class}, then substitute parameters + local class_replaced = std.strReplace(field.value, "{class}", class_name); + param_processor.substitute_parameters(class_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_classes[class_name].class) + ], + + // Builds flow-level processors with parameter substitution + // Processes the 'flow' section of flow classes + build_flow_processors: function(flow_classes, class_name, flow_id, parameters) + [ + [ + // Replace both {class} and {id} in the processor key + local key = std.strReplace( + std.strReplace(processor.key, "{class}", class_name), + "{id}", flow_id + ); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // Replace {class} and {id}, then substitute parameters + local class_replaced = std.strReplace(field.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_classes[class_name].flow) + ], + + // Combines class and flow processors into flow objects + build_flow_objects: function(processor_array) + std.map( + function(item) { + [item[0][0]] +: { + [item[0][1]]: item[1] + } + }, + processor_array + ), + + // Merges all flow objects into a single flows_active configuration + merge_flow_objects: function(flow_objects) + std.foldr( + function(a, b) a + b, + flow_objects, + {} + ), +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/interface-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/interface-builder.jsonnet new file mode 100644 index 00000000..2143e600 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/interface-builder.jsonnet @@ -0,0 +1,30 @@ +// Interface Builder Module +// Processes flow class interfaces with parameter substitution +// Handles both string interfaces and nested object interfaces + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds interfaces for a specific flow class and instance + // Processes the 'interfaces' section of flow classes + build_interfaces: function(flow_classes, class_name, flow_id, parameters) + local interface_spec = flow_classes[class_name].interfaces; + { + [interface.key]: + if std.isString(interface.value) then + // Simple string interface - apply all substitutions + local class_replaced = std.strReplace(interface.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + else + // Complex object interface - process nested fields + { + [field.key]: + local class_replaced = std.strReplace(field.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(interface.value) + } + for interface in std.objectKeysValuesAll(interface_spec) + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/interface-descriptions.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/interface-descriptions.jsonnet new file mode 100644 index 00000000..f9dda8b0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/interface-descriptions.jsonnet @@ -0,0 +1,89 @@ +// Interface Descriptions Module +// Defines all external interfaces available in TrustGraph flows +// These are the 'endpoints' that external systems can interact with + +{ + // Document loading interfaces - for data ingestion + "document-load": { + "description": "Document loader", + "kind": "send", + "visible": true, + }, + "text-load": { + "description": "Text document loader", + "kind": "send", + "visible": true, + }, + + // Data storage interfaces - for processed data streams + "entity-contexts-load": { + "description": "Entity contexts loader", + "kind": "send", + }, + "triples-store": { + "description": "Triples loader", + "kind": "send", + }, + "graph-embeddings-store": { + "description": "Graph embeddings loader", + "kind": "send", + }, + "document-embeddings-store": { + "description": "Document embeddings loader", + "kind": "send", + }, + "objects-store": { + "description": "Object store", + "kind": "request-response", + }, + + // Query interfaces - for retrieving information + "graph-rag": { + "description": "GraphRAG service", + "kind": "request-response", + }, + "document-rag": { + "description": "ChunkRAG service", + "kind": "request-response", + }, + "triples": { + "description": "Triples query service", + "kind": "request-response", + }, + "graph-embeddings": { + "description": "Graph embeddings service", + "kind": "request-response", + }, + "document-embeddings": { + "description": "Document embeddings service", + "kind": "request-response", + }, + "objects": { + "description": "Object query service", + "kind": "request-response", + }, + + // Processing services - for text and data processing + "prompt": { + "description": "Prompt service", + "kind": "request-response", + }, + "agent": { + "description": "Agent service", + "kind": "request-response", + }, + "text-completion": { + "description": "Text completion service", + "kind": "request-response", + }, + + // Query translation services - for natural language queries + "nlp-query": { + "description": "NLP question to GraphQL service", + "kind": "request-response", + }, + "structured-query": { + "description": "Structured query service", + "kind": "request-response", + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/parameter-processor.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/parameter-processor.jsonnet new file mode 100644 index 00000000..f317ae9f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/parameter-processor.jsonnet @@ -0,0 +1,38 @@ +// Parameter Processing Module +// Handles dynamic parameter replacement in configuration values +// Replaces {parameter_name} placeholders with actual parameter values + +{ + // Applies parameter substitutions to string values + // Only processes strings - leaves other types unchanged + substitute_parameters: function(value, parameters) + if std.isString(value) then + std.foldl( + function(acc, param) + // Only do string replacement if param.value is a string + if std.isString(param.value) then + std.strReplace(acc, "{" + param.key + "}", param.value) + else + acc, // Skip replacement for non-string parameter values + std.objectKeysValuesAll(parameters), + value + ) + else + value, + + // Applies parameter substitutions to all values in an object + // Recursively processes nested objects and arrays + substitute_parameters_in_object: function(obj, parameters) + if std.isObject(obj) then + { + [key]: $.substitute_parameters_in_object(obj[key], parameters) + for key in std.objectFields(obj) + } + else if std.isArray(obj) then + [ + $.substitute_parameters_in_object(item, parameters) + for item in obj + ] + else + $.substitute_parameters(obj, parameters), +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/tools.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/tools.jsonnet new file mode 100644 index 00000000..a84cd0bb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/config/tools.jsonnet @@ -0,0 +1,52 @@ +// Tools Configuration Module +// Defines all available tools that can be used by agents and flows +// Each tool specifies its interface, arguments, and behavior + +[ + // Knowledge extraction tool - extracts structured knowledge from text + { + id: "knowledge-extraction", + name: "Knowledge extraction", + description: "Takes a chunk of text and extracts knowledge in definition and relationship formats. The input is a text chunk", + type: "prompt", + template: "agent-kg-extract", + arguments: [ + { + "name": "text", + "type": "string", + "description": "The text chunk", + } + ], + }, + + // Knowledge query tool - queries the knowledge base + { + id: "knowledge-query", + name: "Knowledge query", + description: "This tool queries a knowledge base that holds information about domain-specific information. The question should be a natural language question.", + type: "knowledge-query", + collection: "default", + arguments: [ + { + name: "question", + type: "string", + description: "A simple natural language question.", + } + ] + }, + + // LLM completion tool - general purpose text completion + { + id: "llm-completion", + name: "LLM text completion", + type: "text-completion", + description: "This tool queries an LLM for non-domain-specific information. The question should be a natural language question.", + arguments: [ + { + name: "question", + type: "string", + description: "The question which should be asked of the LLM.", + } + ] + } +] \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/aks-k8s.jsonnet new file mode 100644 index 00000000..07e050f2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/aks-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "disk.csi.azure.com", + parameters: { + // Standard disks (spinning magnetic), Locally Redundant Storage + // Cheapest, basically + skuName: "Standard_LRS", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/docker-compose.jsonnet new file mode 100644 index 00000000..954cde15 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/docker-compose.jsonnet @@ -0,0 +1,230 @@ +{ + + // Extract resources using the engine + package:: function(patterns) + std.foldl( + function(state, p) state + p.create(self), + std.objectValues(patterns), + {} + ), + + container:: function(name) + { + + local container = self, + + name:: name, + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_command:: function(x) self + { command: x }, + + with_runtime:: function(x) self + { runtime: x }, + + with_privileged:: function(x) self + { privileged: x }, + + with_ipc:: function(x) self + { ipc: x }, + + with_capability:: function(x) self + + if std.objectHas(container, "capability") then + { cap_add: container.capability + x } + else + { cap_add: [x], }, + + with_environment:: function(x) self + + if std.objectHas(container, "environment") then + { environment: container.environment + x } + else + { environment: x, }, + + with_device:: function(hdev, cdev) self + + if std.objectHas(container, "devices") then + { devices: container.devices + "%s:%s" % [hdev, cdev] } + else + { devices: [ "%s:%s" % [hdev, cdev] ], }, + + with_limits:: function(c, m) self + { + deploy +: { resources +: { + limits: { cpus: c, memory: m } + } }, + }, + + with_reservations:: function(c, m) self + { + deploy +: { resources +: { + reservations: { cpus: c, memory: m } + } }, + }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [vol.volid, mnt] + ] + else + [ + "%s:%s" % [vol.volid, mnt] + ] + }, + + with_port:: + function(src, dest, name) + self + { + ports: + if std.objectHas(container, "ports") then + container.ports + [ "%d:%d" % [src, dest] ] + else + [ "%d:%d" % [src, dest] ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj.with_environment( + { [x]: "${" + x + "}" } + ), + vars.variables, + self + ), + + restart: "on-failure:100", + + add:: function() { + services +: { + [container.name]: container, + } + } + + }, + + internalService:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + with_size:: function(size) self + { size: size }, + + add:: function() { + volumes +: { + [volume.name]: {} + } + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: "./" + dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + variables:: [], + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + }, + + add:: function() { + } + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.foldl( + function(state, c) state + c.add(), + cont.containers, + {} + ), + + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/eks-k8s.jsonnet new file mode 100644 index 00000000..3fc3f035 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/eks-k8s.jsonnet @@ -0,0 +1,46 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "ebs.csi.aws.com", + parameters: { + type: "gp3", + encrypted: "true", + iops: "6000", + throughput: "400", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/gcp-k8s.jsonnet new file mode 100644 index 00000000..71792426 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/gcp-k8s.jsonnet @@ -0,0 +1,44 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "pd.csi.storage.gke.io", + parameters: { + type: "pd-balanced", + "csi.storage.k8s.io/fstype": "ext4", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/k8s.jsonnet new file mode 100644 index 00000000..686f2029 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/k8s.jsonnet @@ -0,0 +1,358 @@ +{ + + container:: function(name) + { + + local container = self, + + name: name, + limits: {}, + reservations: {}, + ports: [], + volumes: [], + environment: [], + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_command:: function(x) self + { command: x }, + + with_environment:: function(x) self + { + environment: super.environment + [ + { + name: v.key, value: v.value + } + for v in std.objectKeysValues(x) + ], + }, + + with_limits:: function(c, m) self + { limits: { cpu: c, memory: m } }, + + with_reservations:: + function(c, m) self + { reservations: { cpu: c, memory: m } }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: super.volumes + [{ + volume: vol, mount: mnt + }] + }, + + with_port:: + function(src, dest, name) self + { + ports: super.ports + [ + { src: src, dest: dest, name : name } + ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj + { + environment: super.environment + [{ + name: x, + valueFrom: { + secretKeyRef: { + name: vars.name, + key: vars.keyMap[x], + } + } + }] + }, + vars.variables, + self + ), + + add:: function() [ + + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: container.name, + namespace: "trustgraph", + labels: { + app: container.name + } + }, + spec: { + replicas: 1, + selector: { + matchLabels: { + app: container.name, + } + }, + template: { + metadata: { + labels: { + app: container.name, + } + }, + spec: { + containers: [ + { + name: container.name, + image: container.image, + + // FIXME: Make everything run as + // root. Needed to get filesystems + // to be accessible. There's a + // better way of doing this? + securityContext: { + runAsUser: 0, + runAsGroup: 0, + }, + + resources: { + requests: container.reservations, + limits: container.limits + }, + } + ( + if std.length(container.ports) > 0 then + { + ports: [ + { + hostPort: port.src, + containerPort: port.dest, + } + for port in container.ports + ] + } else + {}) + + + (if std.objectHas(container, "command") then + { command: container.command } + else {}) + + + (if std.length(container.environment) > 0 then + { + env: container.environment, + } + else {}) + + + (if std.length(container.volumes) > 0 then + { + volumeMounts: [ + { + mountPath: vol.mount, + name: vol.volume.name, + } + for vol in container.volumes + ] + } + + else + {} + ) + ], + volumes: [ + vol.volume.volRef() + for vol in container.volumes + + ] + } + }, + } + {} + + } + + ] + + }, + + // Just an alias + internalService:: self.service, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + ports: [], + + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + + add:: function() [ + + { + + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + storageClassName: "tg", + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + data: parts + }, + ], + + + volRef:: function() { + name: volume.name, + configMap: { name: volume.name }, + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "Secret", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + data: { + [item.key]: std.base64(item.value) + for item in std.objectKeysValues(parts) + } + }, + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + variables: [], + keyMap: {}, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + }, + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + keyMap: super.keyMap + { [name]: key }, + }, + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.flattenArrays( + [ c.add() for c in cont.containers ] + ), + + }, + + resources:: function(res) + + std.flattenArrays( + [ c.add() for c in res ] + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/minikube-k8s.jsonnet new file mode 100644 index 00000000..858b17ad --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/minikube-k8s.jsonnet @@ -0,0 +1,115 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList, + + volume:: function(name) + { + local volume = self, + name: name, + with_size:: function(size) self + { size: size }, + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolume", + metadata: { + name: volume.name, + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + capacity: { + storage: volume.size, + }, + persistentVolumeReclaimPolicy: "Delete", + hostPath: { + path: "/data/pv-" + volume.name, + }, + } + }, + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + service:: function(containers) + { + local service = self, + name: containers.name, + ports: [], + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + add:: function() [ + { + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + type: "LoadBalancer", + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/noop.jsonnet new file mode 100644 index 00000000..1f384648 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/noop.jsonnet @@ -0,0 +1,79 @@ +{ + + // Extract resources usnig the engine + package:: function(patterns) {}, + + container:: function(name) { + + with_image:: function(x) self + {}, + + with_user:: function(x) self + {}, + + with_command:: function(x) self + {}, + + with_runtime:: function(x) self + {}, + + with_privileged:: function(x) self + {}, + + with_ipc:: function(x) self + {}, + + with_capability:: function(x) self + {}, + + with_environment:: function(x) self + {}, + + with_device:: function(hdev, cdev) self + {}, + + with_limits:: function(c, m) self + {}, + + with_reservations:: function(c, m) self + {}, + + with_volume_mount:: self + {}, + + with_port:: function(src, dest, name) self + {}, + + with_env_var_secrets:: function(vars) self + {}, + + add:: function() {}, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + service:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + volume:: function(name) { + with_size:: function(size) self + {}, + add:: function() {}, + }, + + configVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + secretVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self + {}, + add:: function() {}, + }, + + containers:: function(name, containers) { + add:: function() {}, + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/ovh-k8s.jsonnet new file mode 100644 index 00000000..15d57c83 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/ovh-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "cinder.csi.openstack.org", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", + parameters: { + availability: "nova", + fsType: "ext4", + type: "high-speed", + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/scw-k8s.jsonnet new file mode 100644 index 00000000..eed23968 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/engine/scw-k8s.jsonnet @@ -0,0 +1,40 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "csi.scaleway.com", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/agent-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/agent-extract.jsonnet new file mode 100644 index 00000000..143e736a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/agent-extract.jsonnet @@ -0,0 +1,35 @@ +// Agent-based extraction module +// Uses AI agents for more sophisticated knowledge extraction from text +// Leverages agent tools and reasoning for complex extraction tasks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - internal agent extraction service + "interfaces" +: { + }, + + // No configurable parameters for agent extraction + "parameters" +: { + }, + + // Flow-level processors for agent-based extraction + "flow" +: { + // Agent-based knowledge extraction processor + // Uses AI agents with tools to extract structured knowledge + "kg-extract-agent:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output knowledge triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "agent-request": request("agent:{id}"), // Agent service requests + "agent-response": response("agent:{id}"), // Agent service responses + }, + }, + + // No class-level processors needed + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/agent.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/agent.jsonnet new file mode 100644 index 00000000..19afe660 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/agent.jsonnet @@ -0,0 +1,59 @@ +// Agent management module +// Provides AI agent orchestration and tool integration +// Manages agent conversations, tool calls, and response coordination +// Supports MCP tools, GraphRAG, and structured queries + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // External interfaces for agent operations + "interfaces" +: { + "agent": request_response("agent:{id}"), // Main agent service interface + "mcp-tool": request_response("mcp-tool:{id}"), // MCP tool execution interface + }, + + // No configurable parameters for agent management + "parameters" +: { + }, + + // Flow-level processors for agent management + "flow" +: { + // Agent manager orchestrates agent conversations and tool usage + "agent-manager:{id}": { + // Agent communication channels + request: request("agent:{id}"), // Incoming agent requests + next: request("agent:{id}"), // Multi-turn conversation support + response: response("agent:{id}"), // Agent responses + + // LLM and prompt services + "text-completion-request": request("text-completion:{id}"), // LLM requests + "text-completion-response": response("text-completion:{id}"), // LLM responses + "prompt-request": request("prompt:{id}"), // Prompt processing + "prompt-response": response("prompt:{id}"), + + // Tool integrations + "mcp-tool-request": request("mcp-tool:{id}"), // MCP tool calls + "mcp-tool-response": response("mcp-tool:{id}"), + "graph-rag-request": request("graph-rag:{id}"), // GraphRAG queries + "graph-rag-response": response("graph-rag:{id}"), + "structured-query-request": request("structured-query:{id}"), // Structured data queries + "structured-query-response": response("structured-query:{id}"), + }, + + // MCP tool executor for agent tool usage + "mcp-tool:{id}": { + request: request("mcp-tool:{id}"), // Tool invocation requests + response: response("mcp-tool:{id}"), // Tool execution results + "text-completion-request": request("text-completion:{id}"), // LLM for tool reasoning + "text-completion-response": response("text-completion:{id}"), + }, + + }, + + // Class-level processors for agent-related services + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/documentrag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/documentrag.jsonnet new file mode 100644 index 00000000..65910684 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/documentrag.jsonnet @@ -0,0 +1,100 @@ +// Document RAG (Retrieval Augmented Generation) module +// Implements document-based RAG using chunk embeddings +// Provides semantic search and context-aware question answering +// Supports MCP (Model Context Protocol) tool integration + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces for document RAG functionality + "interfaces" +: { + // Document embedding storage and retrieval + "document-embeddings-store": flow("document-embeddings-store:{id}"), // Embedding storage stream + "document-rag": request_response("document-rag:{id}"), // Main document RAG interface + "document-embeddings": request_response("document-embeddings:{id}"), // Document embedding queries + + // Supporting services + "embeddings": request_response("embeddings:{id}"), // General embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing + "mcp-tool": request_response("mcp-tool:{id}"), // MCP tool integration + "text-completion": request_response("text-completion:{id}"), // LLM text completion + }, + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors for document embedding and storage + "flow" +: { + "document-embeddings:{id}": { + input: flow("chunk-load:{id}"), + output: flow("document-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "de-write:{id}": { + input: flow("document-embeddings-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "document-rag:{id}": { + request: request("document-rag:{id}"), + response: response("document-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "document-embeddings-request": request("document-embeddings:{id}"), + "document-embeddings-response": response("document-embeddings:{id}"), + }, + "de-query:{id}": { + request: request("document-embeddings:{id}"), + response: response("document-embeddings:{id}"), + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "mcp-tool:{id}": { + request: request("mcp-tool:{id}"), + response: response("mcp-tool:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + // Class-level processors for document RAG operations + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/flow-classes.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/flow-classes.jsonnet new file mode 100644 index 00000000..1a1adabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/flow-classes.jsonnet @@ -0,0 +1,105 @@ +// TrustGraph Flow Classes Configuration +// Defines different flow combinations for various use cases +// Each flow class combines multiple functional modules to create complete processing pipelines +// +// Available modules: +// - graphrag: Graph-based RAG with knowledge graphs +// - documentrag: Document-based RAG with chunk embeddings +// - structured: Structured data processing and NLP queries +// - agent: AI agent orchestration and tool integration +// - load: Document loading and preprocessing +// - kg-base: Basic knowledge extraction from text +// - agent-extract: Agent-based knowledge extraction +// - kgcore: Knowledge graph core storage + +// Import all the modular flow components +local graphrag_part = import "graphrag.jsonnet"; +local kg_base_part = import "kg-base.jsonnet"; +local onto_base_part = import "onto-base.jsonnet"; +local agent_extract_part = import "agent-extract.jsonnet"; +local structured_part = import "structured.jsonnet"; +local documentrag_part = import "documentrag.jsonnet"; +local agent_part = import "agent.jsonnet"; +local load_part = import "load.jsonnet"; +local kgcore_part = import "kgcore.jsonnet"; + +{ + + // Complete TrustGraph system with all capabilities + // Includes GraphRAG, DocumentRAG, structured data processing, and knowledge cores + "everything": { + description: "GraphRAG, DocumentRAG, structured data + knowledge cores", + tags: [ + "document-rag", "graph-rag", "knowledge-extraction", + "structured-data", "kgcore" + ], + } + + graphrag_part + documentrag_part + agent_part + load_part + + kg_base_part + structured_part, + + // Dual RAG system without knowledge core creation + // Combines both document and graph-based retrieval + "document-rag+graph-rag": { + description: "Supports GraphRAG and document RAG, no core creation", + tags: ["document-rag", "graph-rag", "knowledge-extraction"], + } + + graphrag_part + documentrag_part + agent_part + load_part + kg_base_part, + + // Graph-based RAG only + // Uses knowledge graphs for context-aware question answering + "graph-rag": { + description: "GraphRAG only", + tags: ["graph-rag", "knowledge-extraction"], + } + + graphrag_part + agent_part + load_part + kg_base_part, + + // Graph-based RAG only + // Uses knowledge graphs for context-aware question answering + "onto-rag": { + description: "Ontology RAG only", + tags: ["graph-rag", "knowledge-extraction"], + } + + graphrag_part + agent_part + load_part + onto_base_part, + + // Document-based RAG only + // Uses document embeddings for semantic search and answers + "document-rag": { + description: "DocumentRAG only", + tags: ["document-rag"], + } + + documentrag_part + load_part, + + // Full RAG system with knowledge core creation + // Includes both RAG types plus persistent knowledge storage + "document-rag+graph-rag+kgcore": { + description: "GraphRAG + DocumentRAG + knowledge core creation", + tags: ["document-rag", "graph-rag", "knowledge-extraction"], + } + + graphrag_part + documentrag_part + agent_part + load_part + + kgcore_part + kg_base_part, + + // GraphRAG with advanced agent-based extraction + // Uses AI agents for sophisticated knowledge extraction + "graph-rag+agent-extract": { + description: "GraphRAG + agent extract", + tags: ["graph-rag", "knowledge-extraction", "agent-extract"], + } + + graphrag_part + agent_part + load_part + agent_extract_part, + + // GraphRAG with structured data processing + // Combines knowledge graphs with structured data queries + "graph-rag+structured-data": { + description: "GraphRAG + structured data", + tags: ["graph-rag", "knowledge-extraction", "structured-data"], + } + + graphrag_part + agent_part + load_part + structured_part, + + // Structured data processing only + // Handles structured data extraction and NLP queries + "structured-data": { + description: "Structured data only", + tags: ["knowledge-extraction", "structured-data"], + } + + agent_part + load_part + structured_part, + +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/graphrag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/graphrag.jsonnet new file mode 100644 index 00000000..94cf5750 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/graphrag.jsonnet @@ -0,0 +1,107 @@ +// GraphRAG flow configuration module +// Implements graph-based retrieval augmented generation (GraphRAG) functionality +// Handles knowledge graph storage, embeddings, and graph-based question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces exposed by the GraphRAG flow + "interfaces" +: { + // Data ingestion interfaces for graph construction + "entity-contexts-load": flow("entity-contexts-load:{id}"), // Entity context data stream + "triples-store": flow("triples-store:{id}"), // RDF triples storage stream + "graph-embeddings-store": flow("graph-embeddings-store:{id}"), // Graph embedding storage + + // Query interfaces for graph-based operations + "graph-rag": request_response("graph-rag:{id}"), // Main GraphRAG query interface + "triples": request_response("triples:{id}"), // Triple store queries + "graph-embeddings": request_response("graph-embeddings:{id}"), // Graph embedding queries + + // Supporting services + "embeddings": request_response("embeddings:{id}"), // General embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing service + "text-completion": request_response("text-completion:{id}"), // LLM text completion + }, + + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors - handle data streams for a specific flow instance + "flow" +: { + "graph-embeddings:{id}": { + input: flow("entity-contexts-load:{id}"), + output: flow("graph-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "triples-write:{id}": { + input: flow("triples-store:{id}"), + }, + "ge-write:{id}": { + input: flow("graph-embeddings-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "graph-rag:{id}": { + request: request("graph-rag:{id}"), + response: response("graph-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "graph-embeddings-request": request("graph-embeddings:{id}"), + "graph-embeddings-response": response("graph-embeddings:{id}"), + "triples-request": request("triples:{id}"), + "triples-response": response("triples:{id}"), + }, + "triples-query:{id}": { + request: request("triples:{id}"), + response: response("triples:{id}"), + }, + "ge-query:{id}": { + request: request("graph-embeddings:{id}"), + response: response("graph-embeddings:{id}"), + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + // Class-level processors - shared across all flow instances of this class + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/helpers.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/helpers.jsonnet new file mode 100644 index 00000000..eabb3bf4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/helpers.jsonnet @@ -0,0 +1,29 @@ +// Helper functions for flow configuration +// Provides utility functions for constructing flow, request, and response URIs +// used throughout the TrustGraph flow configuration system + +// Creates a persistent flow URI for data streams +// Persistent flows retain messages until consumed +local flow(x) = "persistent://tg/flow/" + x; + +// Creates a non-persistent request URI for request-response patterns +// Non-persistent means messages are not retained if no consumer is present +local request(x) = "non-persistent://tg/request/" + x; + +// Creates a non-persistent response URI for request-response patterns +local response(x) = "non-persistent://tg/response/" + x; + +// Creates a request-response pair for bidirectional communication +// Returns an object with both request and response URIs +local request_response(x) = { + request: request(x), + response: response(x), +}; + +// Export all helper functions for use in other modules +{ + flow: flow, + request: request, + response: response, + request_response: request_response, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/kg-base.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/kg-base.jsonnet new file mode 100644 index 00000000..215dd791 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/kg-base.jsonnet @@ -0,0 +1,44 @@ +// Knowledge Graph Base extraction module +// Provides basic knowledge extraction capabilities from text chunks +// Extracts entity definitions and relationships using prompt-based processing + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts entity definitions from text chunks + // Identifies and defines key entities mentioned in the text + "kg-extract-definitions:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output definition triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition extraction prompts + "prompt-response": response("prompt:{id}"), + }, + + // Extracts relationships between entities + // Identifies how entities are connected and interact + "kg-extract-relationships:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output relationship triples + "prompt-request": request("prompt:{id}"), // Relationship extraction prompts + "prompt-response": response("prompt:{id}"), + }, + }, + + // No class-level processors needed + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/kgcore.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/kgcore.jsonnet new file mode 100644 index 00000000..4a857e66 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/kgcore.jsonnet @@ -0,0 +1,31 @@ +// Knowledge Graph Core storage module +// Handles persistent storage of knowledge graph data +// Consolidates triples and graph embeddings into permanent storage +// Creates the core knowledge base for long-term use + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; + +{ + // No external interfaces - internal storage service + "interfaces" +: { + }, + + // No configurable parameters for core storage + "parameters" +: { + }, + + // Flow-level processors for knowledge graph storage + "flow" +: { + // Knowledge graph store consolidates extracted knowledge + // Takes processed triples and embeddings and stores them permanently + "kg-store:{id}": { + "triples-input": flow("triples-store:{id}"), // Input RDF triples stream + "graph-embeddings-input": flow("graph-embeddings-store:{id}"), // Input graph embeddings + }, + }, + + // No class-level processors needed + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/llm-parameters.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/llm-parameters.jsonnet new file mode 100644 index 00000000..a99f72ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/llm-parameters.jsonnet @@ -0,0 +1,59 @@ +{ + + // LLM model selection for normal LLM + "llm-model": { + "type": "llm-model", + "description": "LLM model", + "order": 1, + "advanced": false, + }, + + // LLM model for RAG operations + "llm-rag-model": { + "type": "llm-model", + "description": "LLM model for RAG", + "order": 2, + "advanced": true, + "controlled-by": "llm-model", + }, + + // LLM model selection for normal LLM + "llm-temperature": { + "type": "llm-temperature", + "description": "LLM temperature", + "order": 3, + "advanced": true, + }, + + // LLM model selection for normal LLM + "llm-rag-temperature": { + "type": "llm-temperature", + "description": "LLM temperature for RAG", + "order": 4, + "advanced": true, + }, + + "embeddings-model": { + "type": "embeddings-model", + "description": "Embeddings model", + "order": 5, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-size": { + "type": "chunk-size", + "description": "Chunk size", + "order": 6, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-overlap": { + "type": "chunk-overlap", + "description": "Chunk overlap", + "order": 7, + "advanced": true, + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/load.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/load.jsonnet new file mode 100644 index 00000000..eacf60dd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/load.jsonnet @@ -0,0 +1,50 @@ +// Document loading and preprocessing module +// Handles document ingestion, format conversion, and chunking +// Converts PDFs to text and splits documents into processable chunks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + + // External interfaces for document loading + "interfaces" +: { + "document-load": flow("document-load:{id}"), // Raw document input stream + "text-load": flow("text-document-load:{id}"), // Text document stream + "embeddings": request_response("embeddings:{id}"), // Embedding service for chunks + }, + + // No configurable parameters for document loading + "parameters" +: { + }, + + // Flow-level processors for document preprocessing + "flow" +: { + // PDF decoder converts PDF documents to text + "pdf-decoder:{id}": { + input: flow("document-load:{id}"), // Raw PDF input + output: flow("text-document-load:{id}"), // Extracted text output + }, + + // Chunker splits documents into smaller, processable pieces + "chunker:{id}": { + input: flow("text-document-load:{id}"), // Full text documents + output: flow("chunk-load:{id}"), // Document chunks for processing + "chunk-size": "{chunk-size}", // Chunk size + "chunk-overlap": "{chunk-overlap}", // Overlap between chunks + }, + // Embedding service for converting text chunks to vectors + "embeddings:{id}": { + request: request("embeddings:{id}"), // Embedding requests + response: response("embeddings:{id}"), // Embedding responses + model: "{embeddings-model}", + }, + }, + + // Class-level processors for document loading services + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/onto-base.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/onto-base.jsonnet new file mode 100644 index 00000000..2236f24d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/onto-base.jsonnet @@ -0,0 +1,39 @@ +// Knowledge Graph Base extraction module +// Provides basic knowledge extraction capabilities from text chunks +// Extracts entity definitions and relationships using prompt-based processing + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts using ontology definitions + "kg-extract-ontology:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition + // extraction prompts + "prompt-response": response("prompt:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + + }, + + // No class-level processors needed + "class" +: { + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/ontorag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/ontorag.jsonnet new file mode 100644 index 00000000..94cf5750 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/ontorag.jsonnet @@ -0,0 +1,107 @@ +// GraphRAG flow configuration module +// Implements graph-based retrieval augmented generation (GraphRAG) functionality +// Handles knowledge graph storage, embeddings, and graph-based question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces exposed by the GraphRAG flow + "interfaces" +: { + // Data ingestion interfaces for graph construction + "entity-contexts-load": flow("entity-contexts-load:{id}"), // Entity context data stream + "triples-store": flow("triples-store:{id}"), // RDF triples storage stream + "graph-embeddings-store": flow("graph-embeddings-store:{id}"), // Graph embedding storage + + // Query interfaces for graph-based operations + "graph-rag": request_response("graph-rag:{id}"), // Main GraphRAG query interface + "triples": request_response("triples:{id}"), // Triple store queries + "graph-embeddings": request_response("graph-embeddings:{id}"), // Graph embedding queries + + // Supporting services + "embeddings": request_response("embeddings:{id}"), // General embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing service + "text-completion": request_response("text-completion:{id}"), // LLM text completion + }, + + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors - handle data streams for a specific flow instance + "flow" +: { + "graph-embeddings:{id}": { + input: flow("entity-contexts-load:{id}"), + output: flow("graph-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "triples-write:{id}": { + input: flow("triples-store:{id}"), + }, + "ge-write:{id}": { + input: flow("graph-embeddings-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "graph-rag:{id}": { + request: request("graph-rag:{id}"), + response: response("graph-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "graph-embeddings-request": request("graph-embeddings:{id}"), + "graph-embeddings-response": response("graph-embeddings:{id}"), + "triples-request": request("triples:{id}"), + "triples-response": response("triples:{id}"), + }, + "triples-query:{id}": { + request: request("triples:{id}"), + response: response("triples:{id}"), + }, + "ge-query:{id}": { + request: request("graph-embeddings:{id}"), + response: response("graph-embeddings:{id}"), + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + // Class-level processors - shared across all flow instances of this class + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/structured.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/structured.jsonnet new file mode 100644 index 00000000..45bd35d9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/flows/structured.jsonnet @@ -0,0 +1,108 @@ +// Structured data processing module +// Handles extraction and querying of structured data objects +// Provides natural language to GraphQL query capabilities +// Supports structured data storage and retrieval + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces for structured data operations + "interfaces" +: { + // Supporting services + "embeddings": request_response("embeddings:{id}"), // Embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing + "text-completion": request_response("text-completion:{id}"), // LLM completion + + // Structured data storage and querying + "objects-store": flow("objects-store:{id}"), // Object storage stream + "objects": request_response("objects:{id}"), // Object query service + + // Query interfaces + "nlp-query": request_response("nlp-query:{id}"), // NLP to GraphQL translation + "structured-query": request_response("structured-query:{id}"), // Structured query execution + "structured-diag": request_response("structured-diag:{id}"), // Query diagnostics + }, + + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors for structured data extraction + "flow" +: { + "kg-extract-objects:{id}": { + input: flow("chunk-load:{id}"), + output: flow("objects-store:{id}"), + "entity-contexts": flow("entity-contexts-load:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + "objects-write:{id}": { + input: flow("objects-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "objects-query:{id}": { + request: request("objects:{id}"), + response: response("objects:{id}"), + }, + "nlp-query:{id}": { + request: request("nlp-query:{id}"), + response: response("nlp-query:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + }, + "structured-query:{id}": { + request: request("structured-query:{id}"), + response: response("structured-query:{id}"), + "nlp-query-request": request("nlp-query:{id}"), + "nlp-query-response": response("nlp-query:{id}"), + "objects-query-request": request("objects:{id}"), + "objects-query-response": response("objects:{id}"), + }, + "structured-diag:{id}": { + request: request("structured-diag:{id}"), + response: response("structured-diag:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + // Class-level processors for structured data operations + "class" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/mcp/ddg-mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/mcp/ddg-mcp-server.jsonnet new file mode 100644 index 00000000..bb9157f4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/mcp/ddg-mcp-server.jsonnet @@ -0,0 +1,48 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "ddg-mcp-server-port":: 9870, + + "ddg-mcp-server" +: { + + create:: function(engine) + + local port = $["ddg-mcp-server-port"]; + + local container = + engine.container("ddg-mcp-server") + .with_image(images["ddg-mcp-server"]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "ddg-mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + mcp +:: { + "duckduckgo": { + "remote-name": "search", + local port = $["ddg-mcp-server-port"], + local url = "http://ddg-mcp-server:%s/mcp" % port, + "url": url, + } + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/azure-openai.jsonnet new file mode 100644 index 00000000..0e3ea907 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/azure-openai.jsonnet @@ -0,0 +1,9 @@ +// Azure OpenAI LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-4o", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/azure.jsonnet new file mode 100644 index 00000000..6ffa5fdf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/azure.jsonnet @@ -0,0 +1,9 @@ +// Azure LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/bedrock.jsonnet new file mode 100644 index 00000000..df930e59 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/bedrock.jsonnet @@ -0,0 +1,63 @@ +// AWS Bedrock LLM Model Definitions +// Defines available models and their configurations for AWS Bedrock + +{ + "type": "string", + "description": "LLM model to use", + "default": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + "enum": [ + { + id: "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + description: "Claude Sonnet 4.5 (smartest for complex agents and coding)" + }, + { + id: "global.anthropic.claude-opus-4-5-20251101-v1:0", + description: "Claude Opus 4.5 (maximum intelligence)" + }, + { + id: "global.anthropic.claude-haiku-4-5-20251001-v1:0", + description: "Claude Haiku 4.5 (fastest with near-frontier intelligence)" + }, + { + id: "global.anthropic.claude-opus-4-1-20250805-v1:0", + description: "Claude Opus 4.1 (specialized reasoning)" + }, + { + id: "global.anthropic.claude-sonnet-4-20250514-v1:0", + description: "Claude Sonnet 4.0" + }, + { + id: "global.anthropic.claude-opus-4-20250514-v1:0", + description: "Claude Opus 4.0" + }, + { + id: "anthropic.claude-3-5-haiku-20241022-v1:0", + description: "Claude 3.5 Haiku" + }, + { + id: "anthropic.claude-3-haiku-20240307-v1:0", + description: "Claude 3 Haiku" + }, + { + id: "meta.llama3-1-405b-instruct-v1:0", + description: "Llama 3.1 405B Instruct" + }, + { + id: "meta.llama3-1-70b-instruct-v1:0", + description: "Llama 3.1 70B Instruct" + }, + { + id: "meta.llama3-1-8b-instruct-v1:0", + description: "Llama 3.1 8B Instruct" + }, + { + id: "mistral.mistral-large-2407-v1:0", + description: "Mistral Large" + }, + { + id: "mistral.mixtral-8x7b-instruct-v0:1", + description: "Mixtral 8x7B Instruct" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/chunking-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/chunking-param-types.jsonnet new file mode 100644 index 00000000..f82fa3f7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/chunking-param-types.jsonnet @@ -0,0 +1,25 @@ +// Chunk parameter type definitions + +{ + "chunk-size": { + "type": "integer", + "description": "Chunk size", + "placeholder": 2000, + "helper": "An integer, usually 2000 .. 8000", + "default": 2000, + "min": 0, + "max": 32768, + "required": true + }, + "chunk-overlap": { + "type": "integer", + "description": "Chunk overlap", + "placeholder": 50, + "helper": "An integer, usually 50 .. 100", + "default": 50, + "min": 0, + "max": 8000, + "required": true + }, +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/claude.jsonnet new file mode 100644 index 00000000..40fd434b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/claude.jsonnet @@ -0,0 +1,35 @@ +// Claude LLM Model Definitions +// Defines available models and their configurations for Anthropic's Claude + +{ + "type": "string", + "description": "LLM model to use", + "default": "claude-sonnet-4-5-20250929", + "enum": [ + { + id: "claude-sonnet-4-5-20250929", + description: "Claude Sonnet 4.5 (complex agents + coding)" + }, + { + id: "claude-opus-4-5-20251101", + description: "Claude Opus 4.5 (maximum intelligence)" + }, + { + id: "claude-haiku-4-5-20251001", + description: "Claude Haiku 4.5 (fast)" + }, + { + id: "claude-opus-4-1-20250805", + description: "Claude Opus 4.1 (specialized reasoning)" + }, + { + id: "claude-sonnet-4-20250514", + description: "Claude Sonnet 4.0" + }, + { + id: "claude-3-5-haiku-20241022", + description: "Claude 3.5 Haiku" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/cohere.jsonnet new file mode 100644 index 00000000..5a5865a3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/cohere.jsonnet @@ -0,0 +1,35 @@ +// Cohere LLM Model Definitions +// Defines available models and their configurations for Cohere + +{ + "type": "string", + "description": "LLM model to use", + "default": "command-r-plus-08-2024", + "enum": [ + { + id: "command-r-plus-08-2024", + description: "Command R+ (August 2024)" + }, + { + id: "command-r-08-2024", + description: "Command R (August 2024)" + }, + { + id: "command-r-plus", + description: "Command R+ (legacy)" + }, + { + id: "command-r", + description: "Command R (legacy)" + }, + { + id: "command", + description: "Command" + }, + { + id: "command-light", + description: "Command Light" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..477e25dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/embeddings-fastembed.jsonnet @@ -0,0 +1,127 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "sentence-transformers/all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "BAAI/bge-small-en-v1.5", + "description": "bge-small-en-v1.5" + }, + { + "id": "BAAI/bge-small-zh-v1.5", + "description": "bge-small-zh-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-xs", + "description": "snowflake-arctic-embed-xs" + }, + { + "id": "jinaai/jina-embeddings-v2-small-en", + "description": "jina-embeddings-v2-small-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "snowflake/snowflake-arctic-embed-s", + "description": "snowflake-arctic-embed-s" + }, + { + "id": "BAAI/bge-small-en", + "description": "bge-small-en" + }, + { + "id": "BAAI/bge-base-en-v1.5", + "description": "bge-base-en-v1.5" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "description": "paraphrase-multilingual-MiniLM-L12-v2" + }, + { + "id": "Qdrant/clip-ViT-B-32-text", + "description": "clip-ViT-B-32-text" + }, + { + "id": "jinaai/jina-embeddings-v2-base-de", + "description": "jina-embeddings-v2-base-de" + }, + { + "id": "BAAI/bge-base-en", + "description": "bge-base-en" + }, + { + "id": "snowflake/snowflake-arctic-embed-m", + "description": "snowflake-arctic-embed-m" + }, + { + "id": "thenlper/gte-base", + "description": "gte-base" + }, + { + "id": "jinaai/jina-embeddings-v2-base-en", + "description": "jina-embeddings-v2-base-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1", + "description": "nomic-embed-text-v1" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5", + "description": "nomic-embed-text-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-m-long", + "description": "snowflake-arctic-embed-m-long" + }, + { + "id": "jinaai/jina-clip-v1", + "description": "jina-clip-v1" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + { + "id": "jinaai/jina-embeddings-v2-base-es", + "description": "jina-embeddings-v2-base-es" + }, + { + "id": "jinaai/jina-embeddings-v2-base-code", + "description": "jina-embeddings-v2-base-code" + }, + { + "id": "jinaai/jina-embeddings-v2-base-zh", + "description": "jina-embeddings-v2-base-zh" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "description": "paraphrase-multilingual-mpnet-base-v2" + }, + { + "id": "snowflake/snowflake-arctic-embed-l", + "description": "snowflake-arctic-embed-l" + }, + { + "id": "BAAI/bge-large-en-v1.5", + "description": "bge-large-en-v1.5" + }, + { + "id": "thenlper/gte-large", + "description": "gte-large" + }, + { + "id": "intfloat/multilingual-e5-large", + "description": "multilingual-e5-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/embeddings-huggingface.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/embeddings-huggingface.jsonnet new file mode 100644 index 00000000..32254a2b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/embeddings-huggingface.jsonnet @@ -0,0 +1,31 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "all-mpnet-base-v2", + "description": "all-mpnet-base-v2" + }, + { + "id": "all-distilroberta-v1", + "description": "all-distilroberta-v1" + }, + { + "id": "stsb-bert-large", + "description": "stsb-bert-large" + }, + { + "id": "sentence-camembert-large", + "description": "sentence-camembert-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/embeddings-ollama.jsonnet new file mode 100644 index 00000000..c3f29fbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/embeddings-ollama.jsonnet @@ -0,0 +1,23 @@ +// Embeddings model definitions for Ollama +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "all-minilm:latest", + "enum": [ + { + "id": "all-minilm:latest", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/googleaistudio.jsonnet new file mode 100644 index 00000000..a8891932 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/googleaistudio.jsonnet @@ -0,0 +1,67 @@ +// Google AI Studio LLM Model Definitions +// Defines available models and their configurations for Google AI Studio + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 2.5 models (latest generation) + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-exp", + description: "Gemini 2.0 Flash (experimental)" + }, + + + // Claude models on VertexAI + { + id: "claude-3-5-sonnet@20241022", + description: "Claude 3.5 Sonnet (via VertexAI)" + }, + { + id: "claude-3-5-haiku@20241022", + description: "Claude 3.5 Haiku (via VertexAI)" + }, + { + id: "claude-3-opus@20240229", + description: "Claude 3 Opus (via VertexAI)" + }, + { + id: "claude-3-sonnet@20240229", + description: "Claude 3 Sonnet (via VertexAI)" + }, + { + id: "claude-3-haiku@20240307", + description: "Claude 3 Haiku (via VertexAI)" + }, + + // Llama models on VertexAI + { + id: "llama3-405b-instruct-maas", + description: "Llama 3 405B Instruct (via VertexAI)" + }, + { + id: "llama3-70b-instruct-maas", + description: "Llama 3 70B Instruct (via VertexAI)" + }, + { + id: "llama3-8b-instruct-maas", + description: "Llama 3 8B Instruct (via VertexAI)" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/llamafile.jsonnet new file mode 100644 index 00000000..f6480aa9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/llamafile.jsonnet @@ -0,0 +1,9 @@ +// LlamaFile LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/lmstudio.jsonnet new file mode 100644 index 00000000..1c88da3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/lmstudio.jsonnet @@ -0,0 +1,150 @@ +// LMStudio LLM Model Definitions +// Defines available models and their configurations for LMStudio + +{ + "type": "string", + "description": "LLM model to use", + "default": "llama3.1:70b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/mistral.jsonnet new file mode 100644 index 00000000..07f75b81 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/mistral.jsonnet @@ -0,0 +1,61 @@ +// Mistral LLM Model Definitions +// Defines available models and their configurations for Mistral AI + +{ + "type": "string", + "description": "LLM model to use", + "default": "mistral-medium-2508", + "enum": [ + // Featured models + { + id: "mistral-medium-2508", + description: "Mistral Medium 3.1" + }, + { + id: "mistral-large-2512", + description: "Mistral Large 3" + }, + { + id: "mistral-small-2506", + description: "Mistral Small 3.2" + }, + { + id: "ministral-14b-2512", + description: "Ministral 3 14B" + }, + { + id: "ministral-8b-2512", + description: "Ministral 3 8B" + }, + { + id: "ministral-3b-2512", + description: "Ministral 3 3B" + }, + { + id: "magistral-medium-2509", + description: "Magistral Medium 1.2 (reasoning)" + }, + { + id: "magistral-small-2509", + description: "Magistral Small 1.2 (reasoning)" + }, + { + id: "devstral-2512", + description: "Devstral 2 (code)" + }, + // Other models + { + id: "codestral-2508", + description: "Codestral (code)" + }, + { + id: "pixtral-large-2411", + description: "Pixtral Large (vision)" + }, + { + id: "open-mistral-nemo", + description: "Open Mistral Nemo" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/ollama.jsonnet new file mode 100644 index 00000000..4ef98a6a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/ollama.jsonnet @@ -0,0 +1,150 @@ +// Ollama LLM Model Definitions +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemma3:12b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/openai.jsonnet new file mode 100644 index 00000000..75b2670f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/openai.jsonnet @@ -0,0 +1,31 @@ +// OpenAI LLM Model Definitions +// Defines available models and their configurations for OpenAI's platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-4o", + "enum": [ + { + id: "gpt-4o", + description: "GPT-4o (latest)" + }, + { + id: "gpt-4o-mini", + description: "GPT-4o Mini" + }, + { + id: "gpt-4-turbo", + description: "GPT-4 Turbo" + }, + { + id: "gpt-4", + description: "GPT-4" + }, + { + id: "gpt-3.5-turbo", + description: "GPT-3.5 Turbo" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/temperature-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/temperature-param-types.jsonnet new file mode 100644 index 00000000..f5523e88 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/temperature-param-types.jsonnet @@ -0,0 +1,15 @@ +// LLM temperature definitions + +{ + "llm-temperature": { + "type": "float", + "description": "LLM temperature", + "placeholder": 0.3, + "helper": "A floating point number between 0 and 1", + "default": 0.3, + "min": 0.0, + "max": 10.0, + "required": true + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/vertexai.jsonnet new file mode 100644 index 00000000..6d90f8d3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/vertexai.jsonnet @@ -0,0 +1,68 @@ +// VertexAI LLM Model Definitions +// Defines available models and their configurations for Google's VertexAI platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 2.5 models (latest generation) + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-exp", + description: "Gemini 2.0 Flash (experimental)" + }, + + + // Claude models on VertexAI + { + id: "claude-3-5-sonnet@20241022", + description: "Claude 3.5 Sonnet (via VertexAI)" + }, + { + id: "claude-3-5-haiku@20241022", + description: "Claude 3.5 Haiku (via VertexAI)" + }, + { + id: "claude-3-opus@20240229", + description: "Claude 3 Opus (via VertexAI)" + }, + { + id: "claude-3-sonnet@20240229", + description: "Claude 3 Sonnet (via VertexAI)" + }, + { + id: "claude-3-haiku@20240307", + description: "Claude 3 Haiku (via VertexAI)" + }, + + // Llama models on VertexAI + { + id: "llama3-405b-instruct-maas", + description: "Llama 3 405B Instruct (via VertexAI)" + }, + { + id: "llama3-70b-instruct-maas", + description: "Llama 3 70B Instruct (via VertexAI)" + }, + { + id: "llama3-8b-instruct-maas", + description: "Llama 3 8B Instruct (via VertexAI)" + }, + ], + "required": true +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/vllm.jsonnet new file mode 100644 index 00000000..e372a2ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/parameters/vllm.jsonnet @@ -0,0 +1,125 @@ +// vLLM Model Definitions +// Defines available models and their configurations for vLLM +// vLLM works with any HuggingFace model, these are common optimized choices + +{ + "type": "string", + "description": "LLM model to use", + "default": "meta-llama/Llama-3.1-8B-Instruct", + "enum": [ + // Llama 3.1 models + { + id: "meta-llama/Llama-3.1-8B-Instruct", + description: "Llama 3.1 8B Instruct" + }, + { + id: "meta-llama/Llama-3.1-70B-Instruct", + description: "Llama 3.1 70B Instruct" + }, + { + id: "meta-llama/Llama-3.1-405B-Instruct", + description: "Llama 3.1 405B Instruct" + }, + + // Llama 3.2 models + { + id: "meta-llama/Llama-3.2-1B-Instruct", + description: "Llama 3.2 1B Instruct" + }, + { + id: "meta-llama/Llama-3.2-3B-Instruct", + description: "Llama 3.2 3B Instruct" + }, + + // Qwen2.5 models + { + id: "Qwen/Qwen2.5-0.5B-Instruct", + description: "Qwen2.5 0.5B Instruct" + }, + { + id: "Qwen/Qwen2.5-1.5B-Instruct", + description: "Qwen2.5 1.5B Instruct" + }, + { + id: "Qwen/Qwen2.5-3B-Instruct", + description: "Qwen2.5 3B Instruct" + }, + { + id: "Qwen/Qwen2.5-7B-Instruct", + description: "Qwen2.5 7B Instruct" + }, + { + id: "Qwen/Qwen2.5-14B-Instruct", + description: "Qwen2.5 14B Instruct" + }, + { + id: "Qwen/Qwen2.5-32B-Instruct", + description: "Qwen2.5 32B Instruct" + }, + { + id: "Qwen/Qwen2.5-72B-Instruct", + description: "Qwen2.5 72B Instruct" + }, + + // Mistral models + { + id: "mistralai/Mistral-7B-Instruct-v0.3", + description: "Mistral 7B Instruct v0.3" + }, + { + id: "mistralai/Mixtral-8x7B-Instruct-v0.1", + description: "Mixtral 8x7B Instruct" + }, + { + id: "mistralai/Mixtral-8x22B-Instruct-v0.1", + description: "Mixtral 8x22B Instruct" + }, + + // Phi models + { + id: "microsoft/Phi-3.5-mini-instruct", + description: "Phi 3.5 Mini Instruct" + }, + { + id: "microsoft/Phi-4", + description: "Phi 4" + }, + + // Gemma models + { + id: "google/gemma-2-2b-it", + description: "Gemma 2 2B Instruct" + }, + { + id: "google/gemma-2-9b-it", + description: "Gemma 2 9B Instruct" + }, + { + id: "google/gemma-2-27b-it", + description: "Gemma 2 27B Instruct" + }, + + // DeepSeek models + { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", + description: "DeepSeek-R1 Distill Qwen 1.5B" + }, + { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", + description: "DeepSeek-R1 Distill Qwen 7B" + }, + { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + description: "DeepSeek-R1 Distill Qwen 14B" + }, + { + id: "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + description: "DeepSeek-R1 Distill Qwen 32B" + }, + { + id: "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + description: "DeepSeek-R1 Distill Llama 70B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/agent-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/agent-prompt.txt new file mode 100644 index 00000000..9155d20f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/agent-prompt.txt @@ -0,0 +1,104 @@ +# ReAct Agent System Prompt + +You are an AI assistant that uses the ReAct (Reasoning + Acting) framework to solve problems through systematic reasoning and tool use. + +## Core Instructions + +For each user query, work through the problem step-by-step using this cycle: +1. **Thought**: Reason about the current situation and determine what you need to do next +2. **Action**: Take ONE specific action using an available tool +3. Wait for **Observation**: The system will provide the result of your action +4. Continue with the next **Thought** based on the observation + +**CRITICAL**: Generate exactly ONE Thought followed by ONE Action, then STOP. Do not generate multiple Thought/Action pairs in a single response. Do not generate Observations yourself - the system will provide them. + +## Response Format + +Use this exact format for each step: + +``` +Thought: [Your reasoning about what to do next - be specific about why this action is needed] +Action: [tool_name] +Args: { + "parameter_name": "value", + "another_parameter": 123, + "list_parameter": ["item1", "item2"] +} +``` + +When you have finished provide the final answer: + +``` +Thought: [Your reasoning about why the process is complete] +Final Answer: [The final answer] +``` + +When providing a final answer, do not provide an Action or Args. + +## Action Format Rules + +1. **Tool Name**: Write "Action: " followed by the exact tool name on its own line +2. **Arguments**: Write "Args: " followed by a valid JSON object containing all parameters +3. **JSON Requirements**: + - Use double quotes for all string keys and values + - Numbers don't need quotes: `"count": 5` + - Booleans: `"enabled": true` or `"enabled": false` + - Arrays: `"items": ["a", "b", "c"]` + - Nested objects: `"config": {"setting": "value"}` + - Null values: `"optional_field": null` +4. **Required Parameters**: Include all required parameters for the tool +5. **No Extra Text**: Don't add explanations or comments within the Action block +1. **Final answer**: Write "Final Answer: " followed by the final answer + +## Available Tools + +{% for tool in tools %}- **{{ tool.name }}**: {{ tool.description }} +{% for arg in tool.arguments %} - Required: `"{{ arg.name }}"` ({{ arg.type }}): {{ arg.description }} +{% endfor %} +{% endfor %} + +## Behavior Rules + +1. **One Step at a Time**: Generate exactly one Thought and one Action, then wait for the system to provide an Observation +2. **Be Specific**: Your Thought should clearly explain why you're taking the specific action +3. **Use Context**: Build on previous Observations to inform your next steps +4. **Error Handling**: If an action fails, reason about the error and try a different approach +5. **Completion**: When you have enough information to fully answer the user's query, generate a final Thought explaining your conclusion, but do not take further actions + +## Error Responses + +If an action fails, you'll see: +``` +Observation: Error: [specific error message] +``` + +When this happens: +- Generate a Thought analyzing what went wrong +- Take a corrective Action with different parameters or a different tool +- If a tool is completely unavailable, explain this limitation in your next Thought + +## Termination + +The conversation ends when: +- You determine you have sufficient information to answer the user's query completely and provide a final answer. +- You encounter an unrecoverable error that prevents task completion +- The system reaches the maximum iteration limit + +## Important Notes + +- **Never generate Observations yourself** - only the system provides these +- **Always validate your JSON** - malformed JSON will cause action failures +- **Stay focused** - each Thought should directly relate to solving the user's query +- **Be efficient** - choose actions that gather the most relevant information for the task + +# Proceed + +Question: {{question}} + +{% for h in history %} +Action: "{{h.action}}" +Args: { +{% for k, v in h.arguments.items() %} "{{k}}": "{{v}}" +{% endfor %}} +Observation: "{{h.observation}}" +{% endfor %} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/cohere.jsonnet new file mode 100644 index 00000000..9541e4c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/cohere.jsonnet @@ -0,0 +1,42 @@ +// For Cohere. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/default-prompts.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/default-prompts.jsonnet new file mode 100644 index 00000000..35dda682 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/default-prompts.jsonnet @@ -0,0 +1,234 @@ + +// Prompt templates. For tidy JSONNET use, don't change these templates +// here, but use over-rides in the prompt directory + +{ + + "system-template":: "You are a helpful assistant.", + + "templates":: { + + "question":: { + "prompt": "{{question}}", + }, + + "extract-definitions":: { + "prompt": "\nStudy the following text and derive definitions for any discovered entities.\nDo not provide definitions for entities whose definitions are incomplete\nor unknown.\nOutput relationships in JSON format as an array of objects with fields:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract will be written as plain text. Do not add markdown formatting\nor headers or prefixes. Do not include null or unknown definitions.\n", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "entity": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "entity", + "definition" + ] + } + } + }, + + "extract-relationships":: { + "prompt": "\nStudy the following text and derive entity relationships. For each\nrelationship, derive the subject, predicate and object of the relationship.\nOutput relationships in JSON format as an array of objects with fields:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: false if the object is a simple data type: name, value or date. true if it is an entity.\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract must be written as plain text. Do not add markdown formatting\nor headers or prefixes.\n", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "object": { + "type": "string" + }, + "object-entity": { + "type": "boolean" + }, + }, + "required": [ + "subject", + "predicate", + "object", + "object-entity" + ] + } + } + }, + + "extract-topics":: { + "prompt": "You are a helpful assistant that performs information extraction tasks for a provided text.\nRead the provided text. You will identify topics and their definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The JSON response shall use the following structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "topic": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "topic", + "definition" + ] + } + } + }, + + "extract-rows":: { + "prompt": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "response-type": "json", + }, + + "kg-prompt":: { + "prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "response-type": "text", + }, + + "document-prompt":: { + "prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "response-type": "text", + }, + + "agent-react":: { + "prompt": importstr "agent-prompt.txt", + "response-type": "text" + }, + + "agent-kg-extract":: { + "prompt": "Analyze the following text and extract both entity definitions and relationships. Return the results as JSON with 'definitions' and 'relationships' arrays.\n\nFor definitions, extract entities and their explanations or descriptions.\nFor relationships, extract subject-predicate-object triples where subjects and objects are entities, and predicates are relationship types.\n\nText: {{text}}\n\nReturn JSON only, no other text. Use this exact format:\n{\n \"definitions\": [\n {\n \"entity\": \"entity_name\",\n \"definition\": \"definition_text\"\n }\n ],\n \"relationships\": [\n {\n \"subject\": \"subject_entity\",\n \"predicate\": \"relationship_type\",\n \"object\": \"object_entity_or_literal\",\n \"object-entity\": true\n }\n ]\n}\n", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "definitions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "entity": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "entity", + "definition" + ] + } + }, + "relationships": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "object": { + "type": "string" + }, + "object-entity": { + "type": "boolean" + } + }, + "required": [ + "subject", + "predicate", + "object" + ] + } + } + }, + "required": [ + "definitions", + "relationships" + ] + } + }, + + "schema-selection":: { + "prompt": importstr "schema-selection.txt", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "An array of schema names that are relevant to answering the given question" + } + }, + + "graphql-generation":: { + "prompt": importstr "graphql-generation.txt", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The GraphQL query string generated to answer the question" + }, + "variables": { + "type": "object", + "description": "Object containing any GraphQL variables needed for the query", + "additionalProperties": true + }, + "confidence": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "description": "Float between 0.0-1.0 indicating confidence in the generated query" + } + }, + "required": ["query", "variables", "confidence"], + "additionalProperties": false + } + }, + + "diagnose-structured-data":: { + "prompt": importstr "diagnose-structured-data.txt", + "response-type": "json", + }, + + "diagnose-xml":: { + "prompt": importstr "diagnose-xml.txt", + "response-type": "json", + }, + "diagnose-json":: { + "prompt": importstr "diagnose-json.txt", + "response-type": "json", + }, + "diagnose-csv":: { + "prompt": importstr "diagnose-csv.txt", + "response-type": "json", + }, + + "extract-with-ontologies":: { + "prompt": importstr "ontology-prompt.txt", + "response-type": "json", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-csv.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-csv.txt new file mode 100644 index 00000000..ad9725ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-csv.txt @@ -0,0 +1,370 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in CSV processing and delimiter-separated value formats. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured CSV data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## CSV Processing Expertise + +When working with CSV data, you must: + +1. **Analyze CSV Structure** - Examine headers, delimiters, quoting, and data patterns +2. **Identify Column Mappings** - Map source column names to target fields +3. **Handle Complex CSV Patterns** - Support various CSV formats including: + - Standard comma-separated values: `name,age,city` + - Alternative delimiters: tab-separated (TSV), pipe-separated, semicolon-separated + - Quoted fields with embedded delimiters: `"Last, First",25,"New York, NY"` + - Headers with spaces or special characters: `"Customer Name","Order Date","Total Amount"` + - Files with or without headers + - Multi-line fields with embedded newlines + +## CSV Format Configuration Guidelines + +For CSV format configurations, use these patterns: + +**Basic CSV Configuration:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "skip_rows": 0 + } + } +} +``` + +**Advanced CSV Options:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": "\t", // Tab-separated + "quote_char": "\"", + "escape_char": "\\", + "has_header": true, + "skip_rows": 2, // Skip metadata rows + "null_values": ["", "NULL", "N/A"], + "trim_whitespace": true, + "skip_blank_lines": true + } + } +} +``` + +**CRITICAL: Source Field Names in Mappings** + +When processing CSV files, the parser uses column headers (if present) or generates column indices as field names. Your source field names in mappings must match exactly: + +**CORRECT Example with Headers:** +CSV file: +```csv +Customer Name,Order Date,Total Amount,Status +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "Customer Name": "John Smith", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Customer Name", // βœ… Correct - matches header exactly + "source_field": "Order Date", // βœ… Correct - matches header exactly + "source_field": "Total Amount", // βœ… Correct - matches header exactly + "source_field": "Status" // βœ… Correct - matches header exactly +} +``` + +**CORRECT Example without Headers:** +CSV file without headers uses column indices: +```csv +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "0": "John Smith", + "1": "2024-01-15", + "2": "1000.50", + "3": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "0", // βœ… Correct - first column + "source_field": "1", // βœ… Correct - second column + "source_field": "2", // βœ… Correct - third column + "source_field": "3" // βœ… Correct - fourth column +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - Delimiter character (comma, tab, pipe, semicolon, etc.) + - Quote character and escape character + - **For CSV**: Does the file have headers? Sample structure + - Text encoding (UTF-8, Windows-1252, etc.) + - Any rows to skip (metadata, blank lines) + - How null/empty values are represented + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source column β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Date/time format conversions + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + - Row-level validation rules + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## CSV Structure Analysis + +When presented with CSV data, analyze: + +1. **Delimiter Detection**: What character separates the fields? +2. **Header Presence**: Does the first row contain column names? +3. **Quote Pattern**: Are fields quoted? What quote character is used? +4. **Data Types**: What types are present in each column? +5. **Null Representation**: How are empty/null values represented? +6. **Special Characters**: Are there embedded commas, quotes, or newlines? +7. **Encoding Issues**: Are there any character encoding problems? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + // CSV-specific parsing options + // delimiter, quote_char, has_header, skip_rows, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `split`, `strip_quotes` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_number`, `parse_currency` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `clean_whitespace`, `normalize_encoding` + +**Date/Time Operations:** +- `parse_date`, `format_date`, `date_component` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `numeric_range`, `date_range` + +## CSV-Specific Best Practices + +1. **Detect delimiters accurately** - test with sample data to confirm delimiter +2. **Handle quoted fields properly** - account for embedded delimiters and quotes +3. **Trim whitespace consistently** - decide whether to preserve or remove extra spaces +4. **Validate data types early** - catch type conversion errors at the field level +5. **Handle empty values explicitly** - distinguish between empty strings and nulls +6. **Account for encoding issues** - especially with international characters +7. **Validate row structure** - ensure consistent column counts across rows + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle CSV-specific edge cases** like malformed quotes, inconsistent delimiters +10. **Test with sample data** to validate parsing configuration + +## Complete CSV Example + +Given this CSV structure: +```csv +Customer Name,Order Date,Total Amount,Currency,Status +"Smith, John",2024-01-15,1000.50,USD,Active +"Doe, Jane",2024-01-16,750.25,EUR,Pending +"Johnson, Bob",2024-01-17,,USD,Cancelled +``` + +The parser will: +1. Use `delimiter: ","` and `quote_char: "\""` to parse fields correctly +2. Use `has_header: true` to treat first row as column names +3. Create this parsed data structure for each record: + ```json + { + "Customer Name": "Smith, John", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Currency": "USD", + "Status": "Active" + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "trim_whitespace": true, + "null_values": ["", "NULL"] + } + }, + "mappings": [ + { + "source_field": "Customer Name", // βœ… Matches header exactly + "target_field": "customer_name", + "transforms": [{"type": "trim"}] + }, + { + "source_field": "Order Date", // βœ… Matches header exactly + "target_field": "order_date", + "transforms": [{"type": "to_date", "format": "YYYY-MM-DD"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "Total Amount", // βœ… Matches header exactly + "target_field": "amount", + "transforms": [ + {"type": "to_float"}, + {"type": "default", "value": 0.0} + ] + }, + { + "source_field": "Currency", // βœ… Matches header exactly + "target_field": "currency_code", + "transforms": [{"type": "upper"}], + "validation": [{"type": "in_list", "values": ["USD", "EUR", "GBP"]}] + }, + { + "source_field": "Status", // βœ… Matches header exactly + "target_field": "order_status", + "transforms": [{"type": "lower"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the column headers exactly, or use column indices (0, 1, 2, etc.) for files without headers.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the CSV structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to delimiter detection, header identification, quoting patterns, and data type inference: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-json.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-json.txt new file mode 100644 index 00000000..2fe6341d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-json.txt @@ -0,0 +1,327 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in JSON processing and JSONPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured JSON data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## JSON Processing Expertise + +When working with JSON data, you must: + +1. **Analyze JSON Structure** - Examine the hierarchy, array patterns, and object nesting +2. **Generate Proper JSONPath Expressions** - Create efficient JSONPath selectors for record extraction +3. **Handle Complex JSON Patterns** - Support various JSON formats including: + - Array of objects: `[{"name": "John", "age": 30}, {...}]` + - Nested object arrays: `{"data": {"records": [{"id": 1}, {...}]}}` + - Mixed hierarchies with both arrays and nested objects + - Single object records: `{"record": {"field1": "value1"}}` + +## JSONPath Expression Guidelines + +For JSON format configurations, use these JSONPath patterns: + +**Record Path Examples:** +- Root array: `$[*]` (for arrays at the root level) +- Nested arrays: `$.data.records[*]` or `$.response.items[*]` +- Single object: `$.record` (when there's one record per file) +- Deep nesting: `$.data.results.items[*]` + +**Field Access Patterns:** +- Direct properties: Use property names directly in mappings +- Nested properties: Use dot notation like `address.street` or `contact.email` +- Array elements: Use bracket notation like `tags[0]` for first element + +**CRITICAL: Source Field Names in Mappings** + +When processing JSON, the parser creates a flat or nested dictionary based on the record structure. Your source field names in mappings must match the actual property names in the parsed records: + +**CORRECT Example:** +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "metadata": { + "source": "UN", + "year": 2024 + } +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches property name + "source_field": "Trade (USD)", // βœ… Correct - matches property name + "source_field": "metadata.source", // βœ… Correct - nested property access + "source_field": "metadata.year" // βœ… Correct - nested property access +} +``` + +**JSON Format Configuration Template:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$[*]", // JSONPath to extract records + "flatten_nested": true, // Whether to flatten nested objects + "array_handling": "expand" // How to handle arrays: expand, first, concat + } + } +} +``` + +**Alternative JSON Options:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.items[*]", // For nested array structures + "flatten_nested": false, // Keep nested structure + "null_value_handling": "skip" // skip, empty_string, or preserve + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - JSON structure type (array of objects, nested objects, single records) + - **For JSON**: Sample structure, nesting patterns, array locations + - Sample data or field descriptions + - Any format-specific details (encoding, special null handling, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Nested object flattening requirements + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or null values + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## JSON Structure Analysis + +When presented with JSON data, analyze: + +1. **Root Structure**: Is it an array, object, or nested structure? +2. **Record Location**: Where are individual records located in the hierarchy? +3. **Field Pattern**: How are field names and values structured? + - Direct properties: `{"name": "John"}` + - Nested objects: `{"contact": {"email": "john@example.com"}}` + - Arrays: `{"tags": ["red", "blue"]}` + - Mixed types: `{"data": [{"id": 1, "details": {"name": "John"}}]}` +4. **Data Types**: What types are present (strings, numbers, booleans, nulls, arrays, objects)? +5. **Hierarchy Depth**: How deeply nested are the records and fields? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + // JSON-specific parsing options + // record_path (JSONPath), flatten_nested, array_handling, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `flatten_object`, `extract_array_element`, `join_array` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## JSON-Specific Best Practices + +1. **Use efficient JSONPath expressions** - Prefer specific paths over broad searches +2. **Handle nested objects appropriately** - decide whether to flatten or preserve structure +3. **Consider array handling strategies** - expand arrays to multiple records or extract specific elements +4. **Account for null vs undefined** values in field mappings +5. **Handle mixed data types** within the same field across records +6. **Use appropriate flattening** for deeply nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle JSON-specific edge cases** like empty arrays, null objects, mixed types + +## Complete JSON Example + +Given this JSON structure: +```json +{ + "data": { + "records": [ + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata": { + "source": "World Bank", + "confidence": 0.95 + } + } + ] + } +} +``` + +The parser will: +1. Use `record_path: "$.data.records[*]"` to extract record objects from the array +2. Create this parsed data structure for each record: + ```json + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata.source": "World Bank", // If flattened + "metadata.confidence": 0.95 // If flattened + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.records[*]", + "flatten_nested": true, + "array_handling": "expand" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches property name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches property name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches property name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + }, + { + "source_field": "metadata.source", // βœ… Flattened nested field + "target_field": "data_source" + } + ] +} +``` + +**KEY RULE: source_field names must match the actual property names in the parsed JSON records, using dot notation for nested properties when flattened.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the JSON structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to JSON hierarchy, object patterns, array structures, and generate appropriate JSONPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-structured-data.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-structured-data.txt new file mode 100644 index 00000000..84c4b8be --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-structured-data.txt @@ -0,0 +1,309 @@ + +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` (will be converted to relative paths automatically) +- With namespaces: `//ns:record` or `//soap:Body/data/record` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name + +**CRITICAL: Source Field Names in Mappings** + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary. Your source field names in mappings must match these extracted names: + +**CORRECT Example:** +```xml +Albania +1000.50 +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50" +} +``` + +So your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)" // βœ… Correct - matches parsed field name +} +``` + +**INCORRECT Example:** +```json +{ + "source_field": "Field[@name='Country or Area']", // ❌ Wrong - XPath not needed here + "source_field": "field[@name='Trade (USD)']" // ❌ Wrong - XPath not needed here +} +``` + +**XML Format Configuration Template:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name" // For value pattern + } + } +} +``` + +**Alternative XML Options:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // Direct element-based records + // No field_attribute needed for standard XML + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - File type (CSV, JSON, XML, Excel, fixed-width, etc.) + - **For XML**: Sample structure, namespace prefixes, record element patterns + - Sample data or field descriptions + - Any format-specific details (delimiters, encoding, namespaces, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Record Container**: Where are individual records located? +3. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns +4. **Namespaces**: Are there any namespace prefixes? +5. **Hierarchy Depth**: How deeply nested are the records? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "[csv|json|xml|fixed-width|excel]", + "encoding": "utf-8", + "options": { + // Format-specific parsing options + // For XML: record_path (XPath), field_attribute (if applicable) + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches +2. **Handle namespace prefixes** when present +3. **Identify field attribute patterns** correctly +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes + +## Complete XML Example + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "/ROOT/data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the extracted field names, NOT the XML element structure.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, and generate appropriate XPath expressions: + +{{sample}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-xml.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-xml.txt new file mode 100644 index 00000000..a3d8efa4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/diagnose-xml.txt @@ -0,0 +1,453 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for XML data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured XML data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + - CDATA sections and text nodes + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` +- With namespaces: `//ns:record` or `//soap:Body/data/record` +- Attribute-filtered: `//record[@type='customer']` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name like `field_attribute: "id"` or `field_attribute: "type"` + +## CRITICAL: Source Field Names in Mappings + +The behavior depends on whether you use `field_attribute`: + +### With field_attribute (Attribute-Based Fields) + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary: + +**Example:** +```xml + + Albania + 1000.50 + Active + +``` + +Parser configuration: +```json +{ + "record_path": "//record", + "field_attribute": "name" +} +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)", // βœ… Correct - matches parsed field name + "source_field": "Status" // βœ… Correct - matches parsed field name +} +``` + +### Without field_attribute (Element-Based Fields) + +When NOT using `field_attribute`, use direct element names or XPath expressions: + +**Example:** +```xml + + Albania + 1000.50 + Active + + UN + 2024 + + +``` + +Parser configuration: +```json +{ + "record_path": "//record" + // No field_attribute specified +} +``` + +Your mappings should use: +```json +{ + "source_field": "country", // βœ… Direct element name + "source_field": "trade_amount", // βœ… Direct element name + "source_field": "metadata/source", // βœ… Nested element path + "source_field": "metadata/year" // βœ… Nested element path +} +``` + +## XML Format Configuration Templates + +**For Attribute-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name", // Extract field names from 'name' attribute + "namespace_prefixes": { // Optional: define namespaces + "ns": "http://example.com/namespace" + } + } + } +} +``` + +**For Element-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // XPath to find record elements + "preserve_namespaces": true, // Optional: keep namespace info + "ignore_attributes": false // Optional: include element attributes + } + } +} +``` + +**For Complex XML with Namespaces:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//soap:Body//data:record", + "namespace_prefixes": { + "soap": "http://schemas.xmlsoap.org/soap/envelope/", + "data": "http://example.com/data" + }, + "field_attribute": "name" + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **XML Structure Details** + - Sample XML structure or schema + - Root element and record location + - Field organization (elements vs attributes) + - Namespace declarations and prefixes + - Text encoding (UTF-8, ISO-8859-1, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Namespaces**: Are there namespace declarations? What prefixes are used? +3. **Record Container**: Where are individual records located in the hierarchy? +4. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns: `John` +5. **Data Location**: Are values in element text, attributes, or both? +6. **Hierarchy Depth**: How deeply nested are the records? +7. **Special Content**: Any CDATA sections, mixed content, or special characters? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + // XML-specific parsing options + // record_path (XPath), field_attribute (if applicable), namespace_prefixes + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `strip_cdata`, `decode_html_entities` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_xml_date`, `parse_iso_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `extract_attribute`, `get_text_content` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `valid_xml_name`, `namespace_check` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches like `//` when possible +2. **Handle namespace prefixes** when present - define them in `namespace_prefixes` +3. **Identify field attribute patterns** correctly - determine if using `field_attribute` +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures +7. **Handle CDATA sections** and special characters properly +8. **Preserve or normalize whitespace** as appropriate +9. **Consider XML schema validation** if XSD is available + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle XML-specific edge cases** like empty elements, mixed content, processing instructions + +## Complete XML Examples + +### Example 1: Attribute-Based Fields + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "//data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +### Example 2: Element-Based Fields + +Given this XML structure: +```xml + + + John Smith + john@example.com +
+ 123 Main St + New York +
+ + + 456 + 100.50 + + +
+
+``` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer" + } + }, + "mappings": [ + { + "source_field": "name", // βœ… Direct element name + "target_field": "customer_name" + }, + { + "source_field": "email", // βœ… Direct element name + "target_field": "email_address" + }, + { + "source_field": "address/street", // βœ… Nested element path + "target_field": "street_address" + }, + { + "source_field": "address/city", // βœ… Nested element path + "target_field": "city" + }, + { + "source_field": "@id", // βœ… Attribute reference + "target_field": "customer_id", + "transforms": [{"type": "to_int"}] + } + ] +} +``` + +**KEY RULES:** +- With `field_attribute`: source_field names must match the extracted attribute values +- Without `field_attribute`: use direct element names, nested paths, or XPath expressions +- Use `@attribute` notation for XML element attributes + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, namespace usage, and generate appropriate XPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/gemini.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/gemini.jsonnet new file mode 100644 index 00000000..b9a1e0c0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/gemini.jsonnet @@ -0,0 +1,42 @@ +// For VertexAI Gemini. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/graphql-generation.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/graphql-generation.txt new file mode 100644 index 00000000..58af61dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/graphql-generation.txt @@ -0,0 +1,59 @@ +You are a GraphQL query generation expert. Given a natural language question and provided database schemas, generate a valid GraphQL query embedded in valid JSON. + +## Question: +{{ question }} + +## Provided Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %}- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif %}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif %}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ field.enum_values|join(', ') }}]{% endif %} +{% endfor %} +{% endfor %} + +## GraphQL Query Rules: +1. Use the schema names as GraphQL query fields (e.g., `customers`, `orders`) +2. Apply filters using the `where` parameter with nested filter objects +3. Available filter operators per field type: + - String fields: `eq`, `contains`, `startsWith`, `endsWith`, `in`, `not`, `not_in` + - Integer/Float fields: `eq`, `gt`, `gte`, `lt`, `lte`, `in`, `not`, `not_in` +4. Use `order_by` for sorting (field name as string) +5. Use `direction` for sort direction: `ASC` or `DESC` +6. Use `limit` to restrict number of results +7. Select specific fields in the query body + +## Task Instructions: +1. Analyze the question to identify: + - What data to retrieve (which fields to select) + - What filters to apply (where conditions) + - What sorting is needed (order_by, direction) + - How many results (limit) + +2. Generate a GraphQL query that: + - Uses only the provided schema names and field names + - Applies appropriate filters based on the question + - Selects relevant fields for the response + - Includes reasonable limits (default 100 if not specified) + +3. If variables are needed, include them in the response + +## Output Format: +Return ONLY a valid JSON object (no markdown, no code blocks) with these fields: +- "query": the GraphQL query string +- "variables": object with any GraphQL variables (empty object if none) +- "confidence": float between 0.0-1.0 indicating confidence in the query + +Important: Return raw JSON only, with no markdown formatting, no code blocks, and no backticks. + +## Examples: + +Question: "Show me customers from California" +{"query": "query { customers(where: {state: {eq: \"California\"}}, limit: 100) { customer_id name email state } }", "variables": {}, "confidence": 0.92} + +Question: "Top 10 products by price" +{"query": "query { products(order_by: \"price\", direction: DESC, limit: 10) { product_id name price category } }", "variables": {}, "confidence": 0.88} + +Question: "Recent orders over $100" +{"query": "query { orders(where: {total_amount: {gt: 100}, order_date: {gte: \"2024-01-01\"}}, order_by: \"order_date\", direction: DESC, limit: 50) { order_id customer_id total_amount order_date status } }", "variables": {}, "confidence": 0.96} + +Now generate the GraphQL query for the question above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/mixtral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/mixtral.jsonnet new file mode 100644 index 00000000..cd56e7ef --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/mixtral.jsonnet @@ -0,0 +1,42 @@ +// For Mixtral. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/ontology-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/ontology-prompt.txt new file mode 100644 index 00000000..5a0fcce0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/ontology-prompt.txt @@ -0,0 +1,126 @@ +You are a knowledge extraction expert. Your task is to find entities, relationships, and attributes in text based on a provided schema. + +## Entity Types + +These are the types of entities you should look for: + +{% for class_id, class_def in classes.items() %} +- **{{class_id}}**{% if class_def.subclass_of %} (subclass of {{class_def.subclass_of}}){% endif %}{% if class_def.comment %}: {{class_def.comment}}{% endif %} +{% endfor %} + +## Relationships + +These relationships connect entities to other entities: + +{% for prop_id, prop_def in object_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Attributes + +These attributes describe entity properties (text, numbers, etc.): + +{% for prop_id, prop_def in datatype_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Text to Analyze + +{{text}} + +## Your Task + +Extract the following from the text above: + +1. **Entities**: Things mentioned in the text and their types +2. **Relationships**: How entities relate to each other +3. **Attributes**: Properties of entities (like quantities, descriptions, etc.) + +## Output Format + +Return ONLY valid JSON (no markdown, no code blocks, no backticks) with this structure: + +``` +{ + "entities": [ + { + "entity": "entity name as it appears in text", + "type": "EntityType" + } + ], + "relationships": [ + { + "subject": "subject entity name", + "subject-type": "SubjectType", + "relation": "relationship_name", + "object": "object entity name", + "object-type": "ObjectType" + } + ], + "attributes": [ + { + "entity": "entity name", + "entity-type": "EntityType", + "attribute": "attribute_name", + "value": "literal value" + } + ] +} +``` + +## Important Rules + +1. **Entity names**: Use the exact text as it appears (e.g., "Cornish pasty", "beef") +2. **Types**: Use the EXACT type identifiers from the schema above (e.g., "fo/Recipe", "fo/Food") +3. **Relationships**: Use the EXACT relationship names from the schema (e.g., "fo/has_ingredient") +4. **Attributes**: Use the EXACT attribute names from the schema (e.g., "fo/serves") +5. **Type disambiguation**: If the same entity name appears with different types, list it separately for each type +6. **All arrays can be empty** if nothing is found + +## Example + +Input text: "Cornish pasty is a savory pastry filled with beef and potatoes. This recipe serves 4 people." + +Expected output: +{ + "entities": [ + { + "entity": "Cornish pasty", + "type": "fo/Recipe" + }, + { + "entity": "beef", + "type": "fo/Food" + }, + { + "entity": "potatoes", + "type": "fo/Food" + } + ], + "relationships": [ + { + "subject": "Cornish pasty", + "subject-type": "fo/Recipe", + "relation": "fo/has_ingredient", + "object": "beef", + "object-type": "fo/Food" + }, + { + "subject": "Cornish pasty", + "subject-type": "fo/Recipe", + "relation": "fo/has_ingredient", + "object": "potatoes", + "object-type": "fo/Food" + } + ], + "attributes": [ + { + "entity": "Cornish pasty", + "entity-type": "fo/Recipe", + "attribute": "fo/serves", + "value": "4 people" + } + ] +} + +Now extract entities, relationships, and attributes from the text above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/openai.jsonnet new file mode 100644 index 00000000..5d232337 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/openai.jsonnet @@ -0,0 +1,42 @@ +// For OpenAI LLMs. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/schema-selection.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/schema-selection.txt new file mode 100644 index 00000000..39b180e5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/schema-selection.txt @@ -0,0 +1,25 @@ +You are a database schema selection expert. Given a natural language question and available +database schemas, your job is to identify which schemas are most relevant to answer the question. + +## Available Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}): {{ field.description }} +{% endfor %} + +{% endfor %} + +## Question: +{{ question }} + +## Instructions: +1. Analyze the question to understand what data is being requested +2. Examine each schema to understand what data it contains +3. Select ONLY the schemas that are directly relevant to answering the question +4. Return your answer as a JSON array of schema names + +## Response Format: +Return ONLY a JSON array of schema names, nothing else. +Example: ["customers", "orders", "products"] diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/slm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/slm.jsonnet new file mode 100644 index 00000000..48eb96d0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/prompts/slm.jsonnet @@ -0,0 +1,44 @@ +// For SLM. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/cassandra.jsonnet new file mode 100644 index 00000000..2a9d6d7a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/cassandra.jsonnet @@ -0,0 +1,40 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "cassandra" +: { + + create:: function(engine) + + local vol = engine.volume("cassandra").with_size("20G"); + + local container = + engine.container("cassandra") + .with_image(images.cassandra) + .with_environment({ + JVM_OPTS: "-Xms300M -Xmx300M -Dcassandra.skip_wait_for_gossip_to_settle=0", + }) + .with_limits("1.0", "1000M") + .with_reservations("0.5", "1000M") + .with_port(9042, 9042, "cassandra") + .with_volume_mount(vol, "/var/lib/cassandra"); + + local containerSet = engine.containers( + "cassandra", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9042, 9042, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/falkordb.jsonnet new file mode 100644 index 00000000..78509a43 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/falkordb.jsonnet @@ -0,0 +1,39 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "falkordb" +: { + + create:: function(engine) + + local vol = engine.volume("falkordb").with_size("20G"); + + local container = + engine.container("falkordb") + .with_image(images.falkordb) + .with_limits("1.0", "768M") + .with_reservations("0.5", "768M") + .with_port(6379, 6379, "api") + .with_port(3010, 3000, "ui") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "falkordb", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6379, 6379, "api") + .with_port(3010, 3010, "ui"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/memgraph.jsonnet new file mode 100644 index 00000000..70ad127a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/memgraph.jsonnet @@ -0,0 +1,71 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "memgraph" +: { + + create:: function(engine) + + local vol = engine.volume("memgraph").with_size("20G"); + + local container = + engine.container("memgraph") + .with_image(images.memgraph_mage) + .with_environment({ + MEMGRAPH: "--storage-properties-on-edges=true --storage-enable-edges-metadata=true" + }) + .with_limits("1.0", "1000M") + .with_reservations("0.5", "1000M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/var/lib/memgraph"); + + local containerSet = engine.containers( + "memgraph", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + "memgraph-lab" +: { + + create:: function(engine) + + local container = + engine.container("lab") + .with_image(images.memgraph_lab) + .with_environment({ + QUICK_CONNECT_MG_HOST: "memgraph", + QUICK_CONNECT_MG_PORT: "7687", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3010, 3000, "http"); + + local containerSet = engine.containers( + "lab", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3010, 3010, "http"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/milvus.jsonnet new file mode 100644 index 00000000..1c3e3734 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/milvus.jsonnet @@ -0,0 +1,90 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local minio = import "stores/minio.jsonnet"; + +minio { + + etcd +: { + + create:: function(engine) + + local vol = engine.volume("etcd").with_size("20G"); + + local container = + engine.container("etcd") + .with_image(images.etcd) + .with_command([ + "etcd", + "-advertise-client-urls=http://127.0.0.1:2379", + "-listen-client-urls", + "http://0.0.0.0:2379", + "--data-dir", + "/etcd", + ]) + .with_environment({ + ETCD_AUTO_COMPACTION_MODE: "revision", + ETCD_AUTO_COMPACTION_RETENTION: "1000", + ETCD_QUOTA_BACKEND_BYTES: "4294967296", + ETCD_SNAPSHOT_COUNT: "50000" + }) + .with_limits("1.0", "128M") + .with_reservations("0.25", "128M") + .with_port(2379, 2379, "api") + .with_volume_mount(vol, "/etcd"); + + local containerSet = engine.containers( + "etcd", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(2379, 2379, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + milvus +: { + + create:: function(engine) + + local vol = engine.volume("milvus").with_size("20G"); + + local container = + engine.container("milvus") + .with_image(images.milvus) + .with_command([ + "milvus", "run", "standalone" + ]) + .with_environment({ + ETCD_ENDPOINTS: "etcd:2379", + MINIO_ADDRESS: "minio:9000", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2") + .with_volume_mount(vol, "/var/lib/milvus"); + + local containerSet = engine.containers( + "milvus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/minio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/minio.jsonnet new file mode 100644 index 00000000..8a6aa149 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/minio.jsonnet @@ -0,0 +1,49 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + minio +: { + + create:: function(engine) + + local vol = engine.volume("minio-data").with_size("20G"); + + local container = + engine.container("minio") + .with_image(images.minio) + .with_command([ + "minio", + "server", + "/minio_data", + "--console-address", + ":9001", + ]) + .with_environment({ + MINIO_ROOT_USER: "minioadmin", + MINIO_ROOT_PASSWORD: "minioadmin", + }) + .with_limits("0.5", "128M") + .with_reservations("0.25", "128M") + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console") + .with_volume_mount(vol, "/minio_data"); + + local containerSet = engine.containers( + "minio", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/neo4j.jsonnet new file mode 100644 index 00000000..3a8bb783 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/neo4j.jsonnet @@ -0,0 +1,47 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "neo4j" +: { + + create:: function(engine) + + local vol = engine.volume("neo4j").with_size("20G"); + + local container = + engine.container("neo4j") + .with_image(images.neo4j) + .with_environment({ + NEO4J_AUTH: "neo4j/password", + NEO4J_server_memory_pagecache_size: "512m", + NEO4J_server_memory_heap_max__size: "512m", + // NEO4J_server_bolt_listen__address: "0.0.0.0:7687", + // NEO4J_server_default__listen__address: "0.0.0.0", + // NEO4J_server_http_listen__address: "0.0.0.0:7474", + }) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1536M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "neo4j", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/pinecone.jsonnet new file mode 100644 index 00000000..2bef70fb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/pinecone.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "pinecone" +: { + + create:: function(engine) + + local container = + engine.container("pinecone") + .with_image(images.pinecone) + .with_environment({ + + PORT: "5080", + + INDEX_TYPE: "serverless", + + // Dimension is fixed, 384 is right for miniLM + // sentence embedding + DIMENSION: 384, + + METRIC: "cosine" + + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(5080, 5080, "api"); + + local containerSet = engine.containers( + "pinecone", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(5080, 5080, "api"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/qdrant.jsonnet new file mode 100644 index 00000000..9e807632 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/stores/qdrant.jsonnet @@ -0,0 +1,39 @@ +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; + +{ + + "qdrant" +: { + + create:: function(engine) + + local vol = engine.volume("qdrant").with_size("20G"); + + local container = + engine.container("qdrant") + .with_image(images.qdrant) + .with_limits("1.0", "1024M") + .with_reservations("0.5", "1024M") + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2") + .with_volume_mount(vol, "/qdrant/storage"); + + local containerSet = engine.containers( + "qdrant", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/trustgraph-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/trustgraph-config.jsonnet new file mode 100644 index 00000000..cda29336 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/trustgraph-config.jsonnet @@ -0,0 +1,91 @@ +// TrustGraph Main Configuration +// Clean, modular composition of TrustGraph configuration +// Uses specialized modules for different aspects of config building + +// Import dependencies +local base = import "base/base.jsonnet"; +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local default_prompts = import "prompts/default-prompts.jsonnet"; +local token_costs = import "values/token-costs.jsonnet"; +local flow_classes = import "flows/flow-classes.jsonnet"; +local config_composer = import "config/config-composer.jsonnet"; +local interface_descriptions = import "config/interface-descriptions.jsonnet"; +local tools = import "config/tools.jsonnet"; +local temperature_params = import "parameters/temperature-param-types.jsonnet"; +local chunking_params = import "parameters/chunking-param-types.jsonnet"; + +// Main configuration object +local configuration = { + + // Prompt templates + prompts:: default_prompts, + + // Tool definitions + tools:: tools, + + // MCP configuration + mcp:: {}, + + // Flow classes reference + "flow-classes":: flow_classes, + + // LLM model parameters + "llm-models" +:: {}, + + // Embeddings model parameters + "embeddings-models" +:: {}, + + collections +:: { + "trustgraph:default": { + "user": "default-user", + "collection": "default", + "name": "Default Collection", + "description": "Default collection", + "tags": ["default"], + }, + }, + + // Default model and flow parameters + flow_init_parameters:: { + "llm-model": $["llm-models"].default, + "llm-rag-model": $["llm-models"].default, + "llm-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "llm-rag-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "chunk-size": std.toString($["parameter-types"]["chunk-size"].default), + "chunk-overlap": std.toString($["parameter-types"]["chunk-overlap"].default), + "embeddings-model": $["embeddings-models"].default, + }, + + // Interface descriptions for external endpoints + "interface-descriptions":: interface_descriptions, + + // Parameter type definitions + "parameter-types":: { + "llm-model": $["llm-models"], + "embeddings-model": $["embeddings-models"], + } + chunking_params + temperature_params, + + // Token costs + "token-costs":: token_costs, + + // Build the complete configuration using the composer + configuration:: config_composer.build({ + flow_classes: $["flow-classes"], + default_flow_class: "everything", + default_flow_id: "default", + flow_init_parameters: $["flow_init_parameters"], + prompts: $["prompts"], + tools: $["tools"], + mcp: $["mcp"], + interface_descriptions: $["interface-descriptions"], + parameter_types: $["parameter-types"], + token_costs: $["token-costs"], + collection: $["collections"], + }), + +} + default_prompts; + +// Export the final configuration +configuration diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/util/decode-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/util/decode-config.jsonnet new file mode 100644 index 00000000..503b5b6b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/util/decode-config.jsonnet @@ -0,0 +1,31 @@ + +local components = import "components.jsonnet"; + +local apply = function(p, components) + + local base = { + + with:: function(k, v) self + { + [k]:: v + }, + + with_params:: function(pars) + self + std.foldl( + function(obj, par) obj.with(par.key, par.value), + std.objectKeysValues(pars), + self + ), + + }; + + local component = base + components[p.name]; + + component.with_params(p.parameters); + +local decode = function(config) + local add = function(state, c) state + apply(c, components); + local patterns = std.foldl(add, config, {}); + patterns; + +decode + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/values/images.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/values/images.jsonnet new file mode 100644 index 00000000..e259d58e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/values/images.jsonnet @@ -0,0 +1,32 @@ +local version = import "version.jsonnet"; +{ + cassandra: "docker.io/cassandra:4.1.10", + neo4j: "docker.io/neo4j:2025.08.0-community-bullseye", + pulsar: "docker.io/apachepulsar/pulsar:4.1.0", + pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.4.0", + etcd: "quay.io/coreos/etcd:v3.6.4", + minio: "docker.io/minio/minio:RELEASE.2025-09-07T16-13-09Z", + milvus: "docker.io/milvusdb/milvus:v2.5.17", + prometheus: "docker.io/prom/prometheus:v3.8.0", + grafana: "docker.io/grafana/grafana:12.3.0", + loki: "docker.io/grafana/loki:3.6.2", + trustgraph_base: "docker.io/trustgraph/trustgraph-base:" + version, + trustgraph_flow: "docker.io/trustgraph/trustgraph-flow:" + version, + trustgraph_ocr: "docker.io/trustgraph/trustgraph-ocr:" + version, + trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version, + trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version, + trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version, + trustgraph_mcp: "docker.io/trustgraph/trustgraph-mcp:" + version, + qdrant: "docker.io/qdrant/qdrant:v1.15.4", + memgraph_mage: "docker.io/memgraph/memgraph-mage:3.5", + memgraph_lab: "docker.io/memgraph/lab:3.5.0", + falkordb: "docker.io/falkordb/falkordb:v4.12.5", + "workbench-ui": "docker.io/trustgraph/workbench-ui:1.4.2", + "ddg-mcp-server": "docker.io/trustgraph/ddg-mcp-server:0.1.0", + "tgi-service-intel-xpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-xpu", + "tgi-service-cpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-cpu", + "tgi-service-gaudi": "ghcr.io/huggingface/text-generation-inference:sha-f140440-gaudi", + "vllm-service-intel-xpu": "docker.io/intel/vllm:0.8.0-xpu", + "vllm-service-gaudi": "docker.io/trustgraph/vllm-hpu:027f5645", + "vllm-service-nvidia": "docker.io/vllm/vllm-openai:latest", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/values/token-costs.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/values/token-costs.jsonnet new file mode 100644 index 00000000..84c70373 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/values/token-costs.jsonnet @@ -0,0 +1,102 @@ +{ + "mistral.mistral-large-2407-v1:0": { + "model_name": "mistral.mistral-large-2407-v1:0", + "input_price": 0.000004, + "output_price": 0.000012 + }, + "meta.llama3-1-405b-instruct-v1:0": { + "model_name": "meta.llama3-1-405b-instruct-v1:0", + "input_price": 0.00000532, + "output_price": 0.000016 + }, + "mistral.mixtral-8x7b-instruct-v0:1": { + "model_name": "mistral.mixtral-8x7b-instruct-v0:1", + "input_price": 0.00000045, + "output_price": 0.0000007 + }, + "meta.llama3-1-70b-instruct-v1:0": { + "model_name": "meta.llama3-1-70b-instruct-v1:0", + "input_price": 0.00000099, + "output_price": 0.00000099 + }, + "meta.llama3-1-8b-instruct-v1:0": { + "model_name": "meta.llama3-1-8b-instruct-v1:0", + "input_price": 0.00000022, + "output_price": 0.00000022 + }, + "anthropic.claude-3-haiku-20240307-v1:0": { + "model_name": "anthropic.claude-3-haiku-20240307-v1:0", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "anthropic.claude-3-5-sonnet-20240620-v1:0": { + "model_name": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "cohere.command-r-plus-v1:0": { + "model_name": "cohere.command-r-plus-v1:0", + "input_price": 0.0000030, + "output_price": 0.0000150 + }, + "ollama": { + "model_name": "ollama", + "input_price": 0, + "output_price": 0 + }, + "claude-3-haiku-20240307": { + "model_name": "claude-3-haiku-20240307", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "claude-3-5-sonnet-20240620": { + "model_name": "claude-3-5-sonnet-20240620", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "claude-3-opus-20240229": { + "model_name": "claude-3-opus-20240229", + "input_price": 0.000015, + "output_price": 0.000075 + }, + "claude-3-sonnet-20240229": { + "model_name": "claude-3-sonnet-20240229", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "command-r-08-202": { + "model_name": "command-r-08-202", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "c4ai-aya-23-8b": { + "model_name": "c4ai-aya-23-8b", + "input_price": 0, + "output_price": 0 + }, + "llama.cpp": { + "model_name": "llama.cpp", + "input_price": 0, + "output_price": 0 + }, + "gpt-4o": { + "model_name": "gpt-4o", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-2024-08-06": { + "model_name": "gpt-4o-2024-08-06", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "gpt-4o-2024-05-13": { + "model_name": "gpt-4o-2024-05-13", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-mini": { + "model_name": "gpt-4o-mini", + "input_price": 0.00000015, + "output_price": 0.0000006 + } +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/values/url.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/values/url.jsonnet new file mode 100644 index 00000000..1bacb067 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/values/url.jsonnet @@ -0,0 +1,6 @@ +{ + pulsar: "pulsar://pulsar:6650", + pulsar_admin: "http://pulsar:8080", + milvus: "http://milvus:19530", + qdrant: "http://qdrant:6333", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/zip-readme.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/zip-readme.md new file mode 100644 index 00000000..0b117792 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.7/zip-readme.md @@ -0,0 +1,28 @@ + +Note! this is a subset of possible configurations, to generate your own +launch config use the config util... + +- Production: https://config-ui.demo.trustgraph.ai +- Early release: https://dev.config-ui.demo.trustgraph.ai + +The config util auto-generates deployment instructions for your +configuration, so that's the recommended way to deploy. + +---------------------------------------------------------------------------- + +These are launch configurations for TrustGraph. See https://trustgraph.ai for +the quickstart using docker compose. + +Hint for Linux: There are files here which get mounted as volumes inside +Docker Compose containers. This may trigger SELinux rules on your system, to +permit access insider the containers, use a command like this... + +chcon -Rt svirt_sandbox_file_t grafana/ prometheus/ + +The file vertexai/private.json is a placeholder for real GCP credentials if +you are using the VertexAI LLM. If you're using that in Docker Compose, +replace with your real credentials, and don't forget to permit access if you +are using Linux: + +chcon -Rt svirt_sandbox_file_t vertexai/ + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/README.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/README.md new file mode 100644 index 00000000..23039e9a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/README.md @@ -0,0 +1,125 @@ + +# TrustGraph template generation + +There are two utilities here: + +- `generate`: Generates a single Docker Compose launch configuration + based on configuration you provide. +- `generate-all`: Generates the release bundle for releases. You won't + need to use this unless you are managing releases. + +## `generate-all` + +Previously, this generates a full set of all vector DB / triple store / LLM +combinations, and put them in a single ZIP file. But this got out of +hand, so at the time of writing, this generates a single configuraton +using Qdrant vector DB, Ollama LLM support and Cassandra for a triple store. + +The combinations are contained withing the code, it takes two arguments: +- output ZIP file (is over-written) +- TrustGraph version number + +``` +templates/generate-all output.zip 0.18.11 +``` + +## `generate` + +This utility takes a configuration file describing the components to bundle, +and outputs a Docker Compose YAML file. + +### Input configuration + +The input configuration is a JSON file, an array of components to pull into +the configuration. For each component, there is a name and a (possibly empty) +object describing addtional parameters for that component. + +Example: + +``` +[ + { + "name": "cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "qdrant", + "parameters": {} + }, + { + "name": "embeddings-hf", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph", + "parameters": {} + }, + { + "name": "googleaistudio", + "parameters": { + "googleaistudio-temperature": 0.3, + "googleaistudio-max-output-tokens": 2048, + "googleaistudio-model": "gemini-1.5-pro-002" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 1000, + "chunk-overlap": 50 + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "agent-manager-react", + "parameters": {} + } +] +``` + +If you want to make your own configuration you could try changing the +configuration above: +- Components which are essential: pulsar, trustgraph, graph-rag, grafana, + agent-manager-react +- You need a triple store, one of: cassandra, memgraph, falkordb, neo4j +- You need a vector store, one of: qdrant, pinecone +- You need an LLM, one of: azure, azure-openai, bedrock, claude, cohere, + llamafile, ollama, openai, vertexai. +- You need an embeddings implementation, one of: embeddings-hf, + embeddings-ollama +- Optionally add the Workbench tool: workbench-ui + +Components have over-ridable parameters, look in the component definition +in `templates/components/` to see what you can override. + +### Invocation + +Two parameters: +- The output ZIP file +- The version number + +The configuration file described above is provided on standard input + +``` +templates/generate out.zip 0.18.9 < config.json +``` + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/cassandra.jsonnet new file mode 100644 index 00000000..60f262d3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/cassandra.jsonnet @@ -0,0 +1,50 @@ +local images = import "values/images.jsonnet"; + +{ + + "cassandra" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1000M", + "memory-reservation":: "1000M", + "heap":: "300M", + + create:: function(engine) + + // Capture memory settings into locals + local memLimit = self["memory-limit"]; + local memReserv = self["memory-reservation"]; + local heap = self["heap"]; + + local vol = engine.volume("cassandra").with_size("20G"); + + local container = + engine.container("cassandra") + .with_image(images.cassandra) + .with_environment({ + JVM_OPTS: "-Xms%s -Xmx%s -Dcassandra.skip_wait_for_gossip_to_settle=0" % [ + heap, heap, + ], + }) + .with_limits("1.0", memLimit) + .with_reservations("0.5", memReserv) + .with_port(9042, 9042, "cassandra") + .with_volume_mount(vol, "/var/lib/cassandra"); + + local containerSet = engine.containers( + "cassandra", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9042, 9042, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/ceph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/ceph.jsonnet new file mode 100644 index 00000000..42406a95 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/ceph.jsonnet @@ -0,0 +1,446 @@ + +// +// This majorly does not work +// + +// This configuration fails primarily because it tries to treat Ceph +// like a stateless web app. You are currently pointing mon host to a +// generic service name (ceph-mon), but you aren't telling the Monitor +// process to assume that service identity. + +// To make this work with the "Service Name" approach across any +// engine, you need to fix the binding logic and the messenger protocol. + +// 1. The ceph.conf Fix + +// Need to enable Messenger v2 (modern) and tell the cluster to use +// the service names for its initial quorum. + +//Change this section: +// Ini, TOML + +// [global] +// fsid = %s +// mon initial members = mon0 +// mon host = ceph-mon:6789 +// ... + +// To this: +// Ini, TOML + +// [global] +// fsid = %s +// # Use the actual service names as members +// mon initial members = ceph-mon +// # Explicitly use the Service name (VIP) +// mon host = ceph-mon +// # Force modern protocol +// ms_bind_msgr2 = true +// ms_bind_msgr1 = true + +// 2. The MON Environment & Command Fix + +// This is the most critical part. Your current config sets MON_IP: +// "0.0.0.0". This causes the MON to bind to the Pod IP, which breaks +// when the Pod restarts. You must force it to bind to the Service IP. + +// Update mon_env: +// Code snippet + +// local mon_env = cluster_env + { +// CEPH_DAEMON: "MON", +// MON_NAME: "mon0", +// # Remove MON_IP: "0.0.0.0" +// # Add these: +// MON_ADDR: "ceph-mon", // This says to resolve the service name +// }; + +// Update mon_container command: You are currently wiping the MON data +// on every start (rm -rf /var/lib/ceph/mon/*). Stop doing that. If you +// wipe the data, you lose the cluster state and the OSDs will refuse to +// talk to the "new" MON. + +// Code snippet + +// .with_command([ +// "bash", "-c", +// # 1. Resolve the Service IP at runtime +// "export MON_IP=$(getent hosts ceph-mon | awk '{ print $1 }'); " + +// inject_mon_config + +// # 2. Start the daemon telling it its PUBLIC address is the Service VIP +// "exec /opt/ceph-container/bin/entrypoint.sh" +// ]) + +// 3. Why your current config "Majorly does not work" + +// The "Wipe" Logic: By running rm -rf /var/lib/ceph/mon/* in the +// MON container, you are creating a "New Cluster" every time the +// container starts. Since the OSDs store the fsid and cluster +// secrets, they will reject the "new" MON. + +// DNS Race Condition: Your OSD/MGR/RGW containers wait for +// ceph-mon DNS, which is good. However, if ceph-mon resolves to a +// Round Robin IP (multiple pods) rather than a stable ClusterIP, the +// connection will be flaky. + +// Messenger Protocol: Without ms_bind_msgr2, Ceph defaults to the +// old v1 protocol which is much more sensitive to NAT/Container IP +// mismatches. + +// SUMMARY + +// Component: ceph.conf +// Change: Add ms_bind_msgr2 = true +// Why: Supports modern container networking better. + +// Component: MON Start +// Change: Remove rm -rf +// Why: Ceph MONs must keep their database to maintain the cluster. + +// Component: MON Address +// Change: Use getent hosts ceph-mon +// Why: Forces the MON to advertise the Service VIP instead of its own Pod IP. + +// Component: MON Keyring +// Change: Ensure /etc/ceph/ceph.mon.keyring exists +// Why: MONs need their specific key to start. + +local images = import "values/images.jsonnet"; + +{ + with:: function(key, value) + self + { + ["ceph-" + key]:: value, + }, + + // Ceph credentials and cluster settings + "ceph-access-key":: "object-user", + "ceph-secret-key":: "object-password", + "ceph-cluster-id":: "ceph", + "ceph-fsid":: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + + // Pool redundancy settings + // size: 2 = two replicas for fault tolerance + // min_size: 1 = allow degraded I/O if one OSD is down (prevents cluster freeze) + "ceph-pool-size":: "2", + "ceph-pool-min-size":: "1", + + ceph +: { + create:: function(engine) + // Pre-Shared Cryptographic Material - Config-as-Code Approach + // These keys are generated once and distributed to all daemons + // This ensures cryptographic consistency across the shared-nothing architecture + local admin_key = "AQBpxSBlAAAAABAAU99V6D8vS7Uu9y1S8W0iBg=="; + local mon_key = "AQBpxSBlAAAAABAAn7pL/pG9oT+X6vO7V1S6bg=="; + + // Ceph configuration file - rendered from Jsonnet variables + local ceph_conf = ||| + [global] + fsid = %s + mon initial members = mon0 + mon host = ceph-mon:6789 + public network = 0.0.0.0/0 + cluster network = 0.0.0.0/0 + osd pool default size = %s + osd pool default min size = %s + osd crush chooseleaf type = 0 + auth cluster required = cephx + auth service required = cephx + auth client required = cephx + ||| % [$["ceph-fsid"], $["ceph-pool-size"], $["ceph-pool-min-size"]]; + + // Admin keyring - distributed to all daemons + local admin_keyring = ||| + [client.admin] + key = %s + caps mds = "allow *" + caps mgr = "allow *" + caps mon = "allow *" + caps osd = "allow *" + ||| % [admin_key]; + + // Monitor keyring - used by MON for cluster operations + local mon_keyring = ||| + [mon.] + key = %s + caps mon = "allow *" + ||| % [mon_key]; + + // Config injection command - writes files before entrypoint + local inject_config = "printf '%s' > /etc/ceph/ceph.conf; printf '%s' > /etc/ceph/ceph.client.admin.keyring; " % [ceph_conf, admin_keyring]; + local inject_mon_config = inject_config + ("printf '%s' > /etc/ceph/ceph.mon.keyring; " % [mon_keyring]); + + // Data volumes - sized appropriately for production workloads + local vol_mon = engine.volume("ceph-mon").with_size("20G"); + local vol_mgr = engine.volume("ceph-mgr").with_size("20G"); + local vol_osd = engine.volume("ceph-osd").with_size("100G"); + local vol_rgw = engine.volume("ceph-rgw").with_size("20G"); + + // Isolated config volumes per daemon (ReadWriteOnce compatible) + // Each daemon gets its own non-shared config volume to support + // multi-node scheduling in K8s and other orchestrators + local vol_mon_config = engine.volume("ceph-mon-config").with_size("500M"); + local vol_mgr_config = engine.volume("ceph-mgr-config").with_size("500M"); + local vol_osd_config = engine.volume("ceph-osd-config").with_size("500M"); + local vol_rgw_config = engine.volume("ceph-rgw-config").with_size("500M"); + local vol_init_config = engine.volume("ceph-init-config").with_size("500M"); + + // Simplified cluster environment - Config-as-Code model + // No fetch logic needed - config is injected before entrypoint runs + local cluster_env = { + CLUSTER: $["ceph-cluster-id"], + FSID: $["ceph-fsid"], + KV_TYPE: "none", // No external coordination + }; + + // MON-specific environment + // Config-as-Code: MON uses injected config files, not fetch logic + // + // CRITICAL: MON_DATA_AVAIL="0" forces fresh cluster bootstrap + // The ceph/daemon entrypoint script (variables_stack.sh) uses this as a gate: + // - MON_DATA_AVAIL="0" -> run mkfs, create new cluster with our FSID + // - MON_DATA_AVAIL="1" -> attempt to join existing cluster (infinite probe loop) + // + // Network configuration for monmap generation + local mon_env = cluster_env + { + CEPH_DAEMON: "MON", + MON_NAME: "mon0", + MON_PORT: "6789", + MON_DATA_AVAIL: "0", + MON_IP: "0.0.0.0", + NETWORK_AUTO_DETECT: "4", + CEPH_PUBLIC_NETWORK: "0.0.0.0/0", + }; + + // Simplified daemon environments - Config-as-Code model + // All daemons receive config via injection, not fetch from MON + // This eliminates "static mode" errors and networking complexity + + // MGR-specific environment + local mgr_env = cluster_env + { + CEPH_DAEMON: "MGR", + MGR_NAME: "mgr0", + }; + + // OSD-specific environment + local osd_env = cluster_env + { + CEPH_DAEMON: "OSD", + OSD_TYPE: "directory", + }; + + // RGW-specific environment + local rgw_env = cluster_env + { + CEPH_DAEMON: "RGW", + RGW_NAME: "rgw0", + RGW_FRONTEND_PORT: "7480", + }; + + // MON (Monitor) container - cluster state and quorum + // Config-as-Code: Injects pre-shared keys before entrypoint + // CRITICAL: Wipes /var/lib/ceph/mon/* on every start to force fresh bootstrap + // This ensures MON always uses our FSID and doesn't inherit stale cluster state + local mon_container = + engine.container("ceph-mon") + .with_image(images.ceph) + .with_environment(mon_env) + .with_command([ + "bash", "-c", + "rm -rf /var/lib/ceph/mon/*; " + + inject_mon_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(6789, 6789, "mon") + .with_port(3300, 3300, "mon-msgr2") + .with_volume_mount(vol_mon, "/var/lib/ceph/mon") + .with_volume_mount(vol_mon_config, "/etc/ceph"); + + // MGR (Manager) container - cluster management and dashboard + // Config-as-Code: Uses injected config files with pre-shared keys + // DNS wait ensures MON is available before MGR connects + local mgr_container = + engine.container("ceph-mgr") + .with_image(images.ceph) + .with_environment(mgr_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(7000, 7000, "mgr") + .with_port(8443, 8443, "dashboard") + .with_port(9283, 9283, "prometheus") + .with_volume_mount(vol_mgr, "/var/lib/ceph/mgr") + .with_volume_mount(vol_mgr_config, "/etc/ceph"); + + // OSD (Object Storage Daemon) - actual data storage + // Config-as-Code: Uses injected config files with pre-shared keys + // Increased resources to prevent OOM during recovery operations + // DNS wait ensures MON is available before OSD connects + local osd_container = + engine.container("ceph-osd") + .with_image(images.ceph) + .with_environment(osd_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("2.0", "4096M") + .with_reservations("1.0", "2048M") + .with_port(6800, 6800, "osd") + .with_volume_mount(vol_osd, "/var/lib/ceph/osd") + .with_volume_mount(vol_osd_config, "/etc/ceph"); + + // RGW (RADOS Gateway) - S3 API endpoint + // Config-as-Code: Uses injected config files with pre-shared keys + // DNS wait ensures MON is available before RGW connects + local rgw_container = + engine.container("ceph-rgw") + .with_image(images.ceph) + .with_environment(rgw_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(7480, 7480, "s3") + .with_volume_mount(vol_rgw, "/var/lib/ceph/radosgw") + .with_volume_mount(vol_rgw_config, "/etc/ceph"); + + // Init container - one-time S3 user provisioning + // IMPORTANT: This container exits with code 0 after completion + // Orchestrator must NOT restart it (use K8s Job or Compose restart: "no") + // Config-as-Code: Uses injected config to run radosgw-admin commands + local init_container = + engine.container("ceph-init") + .with_image(images.ceph) + .with_environment({ + CLUSTER: $["ceph-cluster-id"], + FSID: $["ceph-fsid"], + KV_TYPE: "none", + RGW_ACCESS_KEY: $["ceph-access-key"], + RGW_SECRET_KEY: $["ceph-secret-key"], + }) + .with_limits("0.5", "512M") + .with_reservations("0.25", "256M") + .with_volume_mount(vol_init_config, "/etc/ceph") + .with_command([ + "bash", "-c", + inject_config + ||| + set -e + + # Wait for cluster health + echo "Waiting for Ceph cluster to be healthy..." + MAX_ATTEMPTS=60 + ATTEMPT=0 + until ceph --cluster ${CLUSTER} health 2>/dev/null | grep -q "HEALTH_OK\|HEALTH_WARN"; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: Cluster failed to become healthy after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: Cluster not ready, retrying in 5s..." + sleep 5 + done + echo "Cluster is healthy." + + # Wait for RGW availability + echo "Waiting for RGW to be ready..." + ATTEMPT=0 + until curl -sf http://ceph-rgw:7480 >/dev/null 2>&1; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: RGW failed to become ready after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: RGW not ready, retrying in 5s..." + sleep 5 + done + echo "RGW is ready." + + # Idempotent S3 user creation + echo "Provisioning S3 user: ${RGW_ACCESS_KEY}" + if radosgw-admin --cluster ${CLUSTER} user info --uid="${RGW_ACCESS_KEY}" >/dev/null 2>&1; then + echo "User ${RGW_ACCESS_KEY} already exists, skipping creation." + else + echo "Creating new S3 user: ${RGW_ACCESS_KEY}" + radosgw-admin --cluster ${CLUSTER} user create \ + --uid="${RGW_ACCESS_KEY}" \ + --display-name="Object Storage User" \ + --access-key="${RGW_ACCESS_KEY}" \ + --secret-key="${RGW_SECRET_KEY}" + echo "S3 user created successfully." + fi + + echo "Initialization complete. Exiting." + exit 0 + |||, + ]); + + // Container sets - each daemon gets its own for K8s node distribution + local mon_containerSet = engine.containers("ceph-mon", [mon_container]); + local mgr_containerSet = engine.containers("ceph-mgr", [mgr_container]); + local osd_containerSet = engine.containers("ceph-osd", [osd_container]); + local rgw_containerSet = engine.containers("ceph-rgw", [rgw_container]); + local init_containerSet = engine.containers("ceph-init", [init_container]); + + // Services - expose daemon ports for inter-daemon communication + local mon_service = + engine.service(mon_containerSet) + .with_port(6789, 6789, "mon") + .with_port(3300, 3300, "mon-msgr2"); + + local mgr_service = + engine.service(mgr_containerSet) + .with_port(7000, 7000, "mgr") + .with_port(8443, 8443, "dashboard") + .with_port(9283, 9283, "prometheus"); + + local osd_service = + engine.service(osd_containerSet) + .with_port(6800, 6800, "osd"); + + local rgw_service = + engine.service(rgw_containerSet) + .with_port(7480, 7480, "s3"); + + engine.resources([ + + // Data volumes + vol_mon, + vol_mgr, + vol_osd, + vol_rgw, + + // Config volumes (isolated, no sharing) + vol_mon_config, + vol_mgr_config, + vol_osd_config, + vol_rgw_config, + vol_init_config, + + // Container sets + mon_containerSet, + mgr_containerSet, + osd_containerSet, + rgw_containerSet, + init_containerSet, + + // Services + mon_service, + mgr_service, + osd_service, + rgw_service, + + ]) + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/falkordb.jsonnet new file mode 100644 index 00000000..1d4176d8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/falkordb.jsonnet @@ -0,0 +1,38 @@ +local images = import "values/images.jsonnet"; + +{ + + "falkordb" +: { + + create:: function(engine) + + local vol = engine.volume("falkordb").with_size("20G"); + + local container = + engine.container("falkordb") + .with_image(images.falkordb) + .with_limits("1.0", "768M") + .with_reservations("0.5", "768M") + .with_port(6379, 6379, "api") + .with_port(3010, 3000, "ui") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "falkordb", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6379, 6379, "api") + .with_port(3010, 3010, "ui"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/garage.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/garage.jsonnet new file mode 100644 index 00000000..9d339bfb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/garage.jsonnet @@ -0,0 +1,249 @@ +local images = import "values/images.jsonnet"; + +{ + + garage +: { + + // Garage S3 credentials - these are the actual access key ID and secret key + // Access Key ID must be in format: GK + 24 hex characters (12 bytes) + // Secret Key must be 64 hex characters (32 bytes) + // For production, generate secure random values and override these defaults + "access-key":: "GK000000000000000000000001", + "secret-key":: "b171f00be9be4c32c734f4c05fe64c527a8ab5eb823b376cfa8c2531f70fc427", + "rpc-secret":: "bbba746a9e289bad64a9e7a36a4299dac8d6e0b8cc2a6c2937fe756df4492008", + // For a production system, override this value + "admin-token":: "batts-rockhearted-unpartially", + region:: "garage", + "replication-factor":: "1", // Set to 1 for single-node, 3 for production + + // Storage volume sizes + "meta-size":: "2G", // Metadata volume size + "data-size":: "5G", // Data volume size (also used for cluster layout capacity) + + create:: function(engine) + + local accessKey = self["access-key"]; + local secretKey = self["secret-key"]; + local rpcSecret = self["rpc-secret"]; + local adminToken = self["admin-token"]; + local region = self.region; + local replicationFactor = self["replication-factor"]; + local metaSize = self["meta-size"]; + local dataSize = self["data-size"]; + + // Garage daemon configuration file - TOML format + local garage_conf = ||| + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + + db_engine = "lmdb" + + replication_factor = %s + + compression_level = 1 + + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "[::]:3901" + rpc_secret = "%s" + + [s3_api] + s3_region = "%s" + api_bind_addr = "[::]:3900" + root_domain = ".s3.garage.local" + + [s3_web] + bind_addr = "[::]:3902" + root_domain = ".web.garage.local" + index = "index.html" + + [k2v_api] + api_bind_addr = "[::]:3904" + + [admin] + api_bind_addr = "[::]:3903" + admin_token = "%s" + ||| % [replicationFactor, rpcSecret, region, adminToken]; + + // Config volume - contains the rendered garage.toml + local cfgVol = engine.configVolume( + "garage-cfg", "garage", + { + "garage.toml": garage_conf, + } + ); + + // Volumes - Garage stores metadata and data separately + local vol_meta = engine.volume("garage-meta").with_size(metaSize); + local vol_data = engine.volume("garage-data").with_size(dataSize); + + // Main Garage daemon container + local garage_container = + engine.container("garage") + .with_image(images.garage) + .with_command([ + "/garage", "-c", "/etc/garage/garage.toml", "server" + ]) + .with_environment({ + RUST_LOG: "garage=info", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3900, 3900, "s3-api") + .with_port(3901, 3901, "rpc") + .with_port(3902, 3902, "web") + .with_port(3903, 3903, "admin") + .with_port(3904, 3904, "k2v") + .with_volume_mount(cfgVol, "/etc/garage/") + .with_volume_mount(vol_meta, "/var/lib/garage/meta") + .with_volume_mount(vol_data, "/var/lib/garage/data"); + + // Init container - configures cluster layout and creates S3 credentials + // IMPORTANT: This container exits with code 0 after completion + // Orchestrator must NOT restart it (use K8s Job or Compose restart: "no") + // Uses Alpine base image since garage container has no shell + local init_container = + engine.container("garage-init") + .with_image("docker.io/alpine:3.23.2") + .with_environment({ + GARAGE_ACCESS_KEY: accessKey, + GARAGE_SECRET_KEY: secretKey, + GARAGE_REGION: region, + GARAGE_ADMIN_TOKEN: adminToken, + GARAGE_RPC_SECRET: rpcSecret, + GARAGE_DATA_SIZE: dataSize, + }) + .with_limits("0.5", "256M") + .with_reservations("0.25", "128M") + .with_volume_mount(cfgVol, "/etc/garage/") + .with_command([ + "sh", "-c", ||| + set -e + + # Install required tools + echo "Installing curl, jq and downloading garage CLI..." + apk add --no-cache curl jq + + # Download garage binary (v2.1.0) for remote management + curl -fsSL "https://garagehq.deuxfleurs.fr/_releases/v2.1.0/x86_64-unknown-linux-musl/garage" \ + -o /usr/local/bin/garage + chmod +x /usr/local/bin/garage + + echo "Waiting for Garage daemon to be ready..." + MAX_ATTEMPTS=60 + ATTEMPT=0 + # Wait for /health to respond (even 503 is fine - means daemon is up) + until curl -s http://garage:3903/health >/dev/null 2>&1; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: Garage failed to become ready after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: Garage not ready, retrying in 2s..." + sleep 2 + done + echo "Garage daemon is ready." + + # Get the node ID via v2 Admin API + echo "Getting Garage node ID via Admin API..." + curl -s -H "Authorization: Bearer ${GARAGE_ADMIN_TOKEN}" \ + "http://garage:3903/v2/GetNodeInfo?node=self" > /tmp/garage-node-info.json + + # Extract node ID from response (the key in success map is the node ID) + NODE_ID=$(jq -r '.success | to_entries[0].value.nodeId' /tmp/garage-node-info.json) + echo "Node ID: ${NODE_ID}" + + if [ -z "$NODE_ID" ] || [ "$NODE_ID" = "null" ]; then + echo "ERROR: Failed to retrieve node ID" + exit 1 + fi + + # ===== LAYOUT MANAGEMENT VIA REMOTE RPC ===== + # Use garage CLI with -h and -s flags to connect remotely + # -h requires format: @: + # -s provides the RPC secret + + # Construct full RPC identifier + RPC_HOST="${NODE_ID}@garage:3901" + echo "RPC Host: ${RPC_HOST}" + + # Check current layout to see if node is already assigned (idempotent) + echo "Checking current cluster layout..." + LAYOUT_OUTPUT=$(garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" layout show 2>&1) + + # Check if node already has a role assigned + # Layout output shows abbreviated node ID (first 16 chars) + NODE_ID_SHORT="${NODE_ID:0:16}" + if echo "$LAYOUT_OUTPUT" | grep -q "$NODE_ID_SHORT"; then + echo "Node ${NODE_ID_SHORT}... already assigned in layout, skipping." + else + echo "Assigning node to cluster layout..." + # Assign node to zone dc1 with configured capacity + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + layout assign ${NODE_ID} -z dc1 -c ${GARAGE_DATA_SIZE} + + echo "Applying layout configuration..." + # Get current staged version and apply + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + layout apply --version 1 + + echo "Layout configured successfully." + # Wait for layout to stabilize + sleep 5 + fi + + # ===== KEY MANAGEMENT VIA REMOTE RPC ===== + + # Check if key already exists (idempotent) + # GARAGE_ACCESS_KEY is already a valid Garage Key ID (GK + 24 hex chars) + if garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" key info "${GARAGE_ACCESS_KEY}" >/dev/null 2>&1; then + echo "Access key ${GARAGE_ACCESS_KEY} already exists, skipping creation." + else + echo "Importing S3 access key: ${GARAGE_ACCESS_KEY}" + + # Import key with the provided credentials (both already in valid Garage format) + # GARAGE_ACCESS_KEY = Key ID (GK + 24 hex chars) + # GARAGE_SECRET_KEY = Secret (64 hex chars) + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + key import "${GARAGE_ACCESS_KEY}" "${GARAGE_SECRET_KEY}" --yes + + echo "Access key imported successfully." + fi + + # Grant permissions to the key + echo "Granting create-bucket permission to key..." + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + key allow --create-bucket "${GARAGE_ACCESS_KEY}" + + echo "" + echo "Garage initialization complete!" + echo "S3 Endpoint: http://garage:3900" + echo "Region: ${GARAGE_REGION}" + exit 0 + |||, + ]); + + // Container sets + local garage_containerSet = engine.containers("garage", [garage_container]); + local init_containerSet = engine.containers("garage-init", [init_container]); + + // Service - expose Garage ports + local garage_service = + engine.service(garage_containerSet) + .with_port(3900, 3900, "s3-api") + .with_port(3901, 3901, "rpc") + .with_port(3902, 3902, "web") + .with_port(3903, 3903, "admin") + .with_port(3904, 3904, "k2v"); + + engine.resources([ + cfgVol, + vol_meta, + vol_data, + garage_containerSet, + init_containerSet, + garage_service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/memgraph.jsonnet new file mode 100644 index 00000000..eeed2e4e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/memgraph.jsonnet @@ -0,0 +1,70 @@ +local images = import "values/images.jsonnet"; + +{ + + "memgraph" +: { + + create:: function(engine) + + local vol = engine.volume("memgraph").with_size("20G"); + + local container = + engine.container("memgraph") + .with_image(images.memgraph_mage) + .with_environment({ + MEMGRAPH: "--storage-properties-on-edges=true --storage-enable-edges-metadata=true" + }) + .with_limits("1.0", "1000M") + .with_reservations("0.5", "1000M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/var/lib/memgraph"); + + local containerSet = engine.containers( + "memgraph", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + "memgraph-lab" +: { + + create:: function(engine) + + local container = + engine.container("lab") + .with_image(images.memgraph_lab) + .with_environment({ + QUICK_CONNECT_MG_HOST: "memgraph", + QUICK_CONNECT_MG_PORT: "7687", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3010, 3000, "http"); + + local containerSet = engine.containers( + "lab", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3010, 3010, "http"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/milvus.jsonnet new file mode 100644 index 00000000..5bed3820 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/milvus.jsonnet @@ -0,0 +1,89 @@ +local images = import "values/images.jsonnet"; +local minio = import "backends/minio.jsonnet"; + +minio { + + etcd +: { + + create:: function(engine) + + local vol = engine.volume("etcd").with_size("20G"); + + local container = + engine.container("etcd") + .with_image(images.etcd) + .with_command([ + "etcd", + "-advertise-client-urls=http://127.0.0.1:2379", + "-listen-client-urls", + "http://0.0.0.0:2379", + "--data-dir", + "/etcd", + ]) + .with_environment({ + ETCD_AUTO_COMPACTION_MODE: "revision", + ETCD_AUTO_COMPACTION_RETENTION: "1000", + ETCD_QUOTA_BACKEND_BYTES: "4294967296", + ETCD_SNAPSHOT_COUNT: "50000" + }) + .with_limits("1.0", "128M") + .with_reservations("0.25", "128M") + .with_port(2379, 2379, "api") + .with_volume_mount(vol, "/etcd"); + + local containerSet = engine.containers( + "etcd", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(2379, 2379, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + milvus +: { + + create:: function(engine) + + local vol = engine.volume("milvus").with_size("20G"); + + local container = + engine.container("milvus") + .with_image(images.milvus) + .with_command([ + "milvus", "run", "standalone" + ]) + .with_environment({ + ETCD_ENDPOINTS: "etcd:2379", + MINIO_ADDRESS: "minio:9000", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2") + .with_volume_mount(vol, "/var/lib/milvus"); + + local containerSet = engine.containers( + "milvus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/minio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/minio.jsonnet new file mode 100644 index 00000000..b38bb81f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/minio.jsonnet @@ -0,0 +1,48 @@ +local images = import "values/images.jsonnet"; + +{ + + minio +: { + + create:: function(engine) + + local vol = engine.volume("minio-data").with_size("20G"); + + local container = + engine.container("minio") + .with_image(images.minio) + .with_command([ + "minio", + "server", + "/minio_data", + "--console-address", + ":9001", + ]) + .with_environment({ + MINIO_ROOT_USER: "minioadmin", + MINIO_ROOT_PASSWORD: "minioadmin", + }) + .with_limits("0.5", "128M") + .with_reservations("0.25", "128M") + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console") + .with_volume_mount(vol, "/minio_data"); + + local containerSet = engine.containers( + "minio", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/neo4j.jsonnet new file mode 100644 index 00000000..46c61e0f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/neo4j.jsonnet @@ -0,0 +1,46 @@ +local images = import "values/images.jsonnet"; + +{ + + "neo4j" +: { + + create:: function(engine) + + local vol = engine.volume("neo4j").with_size("20G"); + + local container = + engine.container("neo4j") + .with_image(images.neo4j) + .with_environment({ + NEO4J_AUTH: "neo4j/password", + NEO4J_server_memory_pagecache_size: "512m", + NEO4J_server_memory_heap_max__size: "512m", + // NEO4J_server_bolt_listen__address: "0.0.0.0:7687", + // NEO4J_server_default__listen__address: "0.0.0.0", + // NEO4J_server_http_listen__address: "0.0.0.0:7474", + }) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1536M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "neo4j", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/pinecone.jsonnet new file mode 100644 index 00000000..2bef70fb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/pinecone.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "pinecone" +: { + + create:: function(engine) + + local container = + engine.container("pinecone") + .with_image(images.pinecone) + .with_environment({ + + PORT: "5080", + + INDEX_TYPE: "serverless", + + // Dimension is fixed, 384 is right for miniLM + // sentence embedding + DIMENSION: 384, + + METRIC: "cosine" + + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(5080, 5080, "api"); + + local containerSet = engine.containers( + "pinecone", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(5080, 5080, "api"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/qdrant.jsonnet new file mode 100644 index 00000000..2e6761f8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/backends/qdrant.jsonnet @@ -0,0 +1,65 @@ +local images = import "values/images.jsonnet"; + +{ + + "qdrant" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1024M", + "memory-reservation":: "1024M", + + // Mmap settings for low-memory mode (trades latency for memory) + // Set to null to disable, or string value to enable + "memmap-threshold-kb":: null, + "on-disk-payload":: null, + + create:: function(engine) + + // Capture memory settings into locals + local memLimit = self["memory-limit"]; + local memReserv = self["memory-reservation"]; + local mmapThreshold = self["memmap-threshold-kb"]; + local onDiskPayload = self["on-disk-payload"]; + + local vol = engine.volume("qdrant").with_size("20G"); + + // Build environment with optional mmap settings + local baseEnv = {}; + local env = baseEnv + + (if mmapThreshold != null then { + QDRANT__STORAGE__MEMMAP_THRESHOLD_KB: mmapThreshold, + } else {}) + + (if onDiskPayload != null then { + QDRANT__STORAGE__ON_DISK_PAYLOAD: onDiskPayload, + } else {}); + + local container = + engine.container("qdrant") + .with_image(images.qdrant) + .with_limits("1.0", memLimit) + .with_reservations("0.5", memReserv) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2") + .with_volume_mount(vol, "/qdrant/storage") + + (if std.length(env) > 0 then { + environment+: env, + } else {}); + + local containerSet = engine.containers( + "qdrant", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/components.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/components.jsonnet new file mode 100644 index 00000000..03a55208 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/components.jsonnet @@ -0,0 +1,121 @@ +{ + + // Essentials + "trustgraph-base": import "core/trustgraph.jsonnet", + "rev-gateway": import "core/rev-gateway.jsonnet", + "pulsar": import "pulsar/pulsar.jsonnet", + + // LLMs + "azure": import "llm/azure.jsonnet", + "azure-openai": import "llm/azure-openai.jsonnet", + "bedrock": import "llm/bedrock.jsonnet", + "claude": import "llm/claude.jsonnet", + "cohere": import "llm/cohere.jsonnet", + "googleaistudio": import "llm/googleaistudio.jsonnet", + "llamafile": import "llm/llamafile.jsonnet", + "lmstudio": import "llm/lmstudio.jsonnet", + "mistral": import "llm/mistral.jsonnet", + "ollama": import "llm/ollama.jsonnet", + "openai": import "llm/openai.jsonnet", + "vertexai": import "llm/vertexai.jsonnet", + "tgi": import "llm/tgi.jsonnet", + "vllm": import "llm/vllm.jsonnet", + + // Embeddings + "embeddings-ollama": import "embeddings/embeddings-ollama.jsonnet", + "embeddings-hf": import "embeddings/embeddings-hf.jsonnet", + "embeddings-fastembed": import "embeddings/embeddings-fastembed.jsonnet", + + // OCR options + "ocr": import "ocr/ocr.jsonnet", + "mistral-ocr": import "ocr/mistral-ocr.jsonnet", + + // Vector stores + "vector-store-milvus": import "vector-store/milvus.jsonnet", + "vector-store-qdrant": import "vector-store/qdrant.jsonnet", + "vector-store-pinecone": import "vector-store/pinecone.jsonnet", + + // Triples stores + "triple-store-cassandra": import "triple-store/cassandra.jsonnet", + "triple-store-neo4j": import "triple-store/neo4j.jsonnet", + "triple-store-falkordb": import "triple-store/falkordb.jsonnet", + "triple-store-memgraph": import "triple-store/memgraph.jsonnet", + + // Object stores + "row-store-cassandra": import "row-store/cassandra.jsonnet", + + // Observability support + "grafana": import "monitoring/grafana.jsonnet", + "loki": import "monitoring/loki.jsonnet", + + // Pulsar manager is a UI for Pulsar. Uses a LOT of memory + "pulsar-manager": import "pulsar/pulsar-manager.jsonnet", + + "override-recursive-chunker": import "core/chunker-recursive.jsonnet", + + // The prompt manager + "prompt-overrides": import "core/prompt-overrides.jsonnet", + + // Extra MCP services + "ddg-mcp-server": import "mcp/ddg-mcp-server.jsonnet", + + // Does nothing. But, can be a hack to overwrite parameters + "null": {}, + + // Passthrough: returns parameters directly, preserving +: merge syntax + // Also supports JSON-safe routing with prefixed parameters like "cassandra-heap" + "override": { + local route = function(target) + function(prefix, k, v) + local suffix = std.substr(k, std.length(prefix), std.length(k) - std.length(prefix)); + { [target] +: { [suffix]:: v } }, + + local routes = { + "cassandra-": route("cassandra"), + "pulsar-": route("pulsar"), + "qdrant-": route("qdrant"), + "api-gateway-": route("api-gateway"), + "librarian-": route("librarian"), + }, + + with_params:: function(pars) + std.foldl( + function(acc, k) + local matchingPrefixes = [p for p in std.objectFields(routes) if std.startsWith(k, p)]; + if std.length(matchingPrefixes) > 0 then + local prefix = matchingPrefixes[0]; + acc + routes[prefix](prefix, k, pars[k]) + else + acc + { [k]:: pars[k] }, + std.objectFields(pars), + {} + ), + }, + + // Memory profiles + "memory-profile-low": import "profiles/memory-profile-low.jsonnet", + + // Model hosting + + "hosting-intel-battlemage-vllm": + import "model-hosting/intel-battlemage-vllm.jsonnet", + + "hosting-cpu-tgi": + import "model-hosting/cpu-tgi.jsonnet", + + "hosting-intel-xpu-tgi": + import "model-hosting/intel-xpu-tgi.jsonnet", + + "hosting-intel-gaudi-tgi": + import "model-hosting/intel-gaudi-tgi.jsonnet", + + "hosting-intel-xpu-vllm": + import "model-hosting/intel-xpu-vllm.jsonnet", + + "hosting-intel-gaudi-vllm": + import "model-hosting/intel-gaudi-vllm.jsonnet", + + "hosting-nvidia-gpu-vllm": + import "model-hosting/nvidia-gpu-vllm.jsonnet", + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/agent-manager-react.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/agent-manager-react.jsonnet new file mode 100644 index 00000000..954d80a7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/agent-manager-react.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "agent-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("agent-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "agent-manager-react", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "agent-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/chunker-recursive.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/chunker-recursive.jsonnet new file mode 100644 index 00000000..0b924aba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/chunker-recursive.jsonnet @@ -0,0 +1,55 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "chunk-size":: 2000, + "chunk-overlap":: 100, + + "chunker" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + std.toString($["chunk-size"]), + "--chunk-overlap", + std.toString($["chunk-overlap"]), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/configuration.jsonnet new file mode 100644 index 00000000..75afebb9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/configuration.jsonnet @@ -0,0 +1,57 @@ + +// This puts the default configuration together. References many things, +// flow classes, a default flow, token costs, prompts, agent tools + +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "init-trustgraph" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local cfgVol = engine.configVolume( + "trustgraph-cfg", "trustgraph", + { + "config.json": importstr "trustgraph/config.json", + } + ); + + local container = + engine.container("init-trustgraph") + .with_image(images.trustgraph_flow) + .with_command( + [ + "tg-init-trustgraph", + "-p", + url.pulsar_admin, + "--config-file", + "/trustgraph/config.json", + ] + ) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_volume_mount(cfgVol, "/trustgraph/"); + + local containerSet = engine.containers( + "init-trustgraph", [ container ] + ); + + engine.resources([ + cfgVol, + containerSet, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/document-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/document-rag.jsonnet new file mode 100644 index 00000000..e466e3a4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/document-rag.jsonnet @@ -0,0 +1,92 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "document-rag" +: { + + "doc-limit":: 20, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local docLimit = self["doc-limit"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "document-rag", + "-p", + url.pulsar, + "--doc-limit", + std.toString(docLimit), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "document-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "document-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/graph-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/graph-rag.jsonnet new file mode 100644 index 00000000..718441ea --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/graph-rag.jsonnet @@ -0,0 +1,283 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "kg-extract-definitions" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-definitions") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-definitions", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-definitions", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-relationships" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-relationships") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-relationships", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-relationships", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-agent" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-agent") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-agent", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-agent", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-ontology" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "300M", + "memory-reservation":: "300M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-ontology") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-ontology", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-ontology", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-rag" +: { + + concurrency:: 1, + "entity-limit":: 50, + "triple-limit":: 30, + "max-subgraph-size":: 400, + "max-path-length":: 2, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local entityLimit = self["entity-limit"]; + local tripleLimit = self["triple-limit"]; + local maxSubgraphSize = self["max-subgraph-size"]; + local maxPathLength = self["max-path-length"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("graph-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-rag", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--entity-limit", + std.toString(entityLimit), + "--triple-limit", + std.toString(tripleLimit), + "--max-subgraph-size", + std.toString(maxSubgraphSize), + "--max-path-length", + std.toString(maxPathLength), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "graph-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/librarian.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/librarian.jsonnet new file mode 100644 index 00000000..2ecda5a8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/librarian.jsonnet @@ -0,0 +1,58 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local garage = import "backends/garage.jsonnet"; +local cassandra = import "backends/cassandra.jsonnet"; + +{ + + "librarian" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("librarian") + .with_image(images.trustgraph_flow) + .with_command([ + "librarian", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + "--object-store-endpoint", + url.object_store, + "--object-store-access-key", + $.garage["access-key"], + "--object-store-secret-key", + $.garage["secret-key"], + "--object-store-region", + $.garage.region, + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "librarian", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +// Garage and Cassandra are used by the Librarian +} + garage + cassandra + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/mcp-server.jsonnet new file mode 100644 index 00000000..9bf7e3bd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/mcp-server.jsonnet @@ -0,0 +1,54 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "mcp-server" +: { + + port:: 8000, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local port = self.port; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("mcp-server-secret") + .with_env_var("MCP_SERVER_SECRET", "mcp-server-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local container = + engine.container("mcp-server") + .with_image(images.trustgraph_mcp) + .with_command([ + "mcp-server", + "--port", + std.toString(port), + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/prompt-overrides.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/prompt-overrides.jsonnet new file mode 100644 index 00000000..852ec09d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/prompt-overrides.jsonnet @@ -0,0 +1,24 @@ +local default_prompts = import "prompts/default-prompts.jsonnet"; + +{ + + with:: function(key, value) + if (key == "system-template") then + self + { + prompts +:: { + "system-template": value, + } + } + else + self + { + prompts +:: { + templates +:: { + [key] +:: { + prompt: value + } + } + } + }, + +} + default_prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/prompt-template.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/prompt-template.jsonnet new file mode 100644 index 00000000..3cd79d59 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/prompt-template.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "prompt" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("prompt") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "prompt", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "prompt-rag" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("prompt-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--id", + "prompt-rag", + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "prompt-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/rev-gateway.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/rev-gateway.jsonnet new file mode 100644 index 00000000..fbd9f77a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/rev-gateway.jsonnet @@ -0,0 +1,62 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "rev-gateway" +: { + + // Invalid, but at least means the rev-gateway won't connect to anything + // it shouldn't. + token:: "INVALID_TOKEN", + uri:: "wss://127.0.0.1/api/v1/relay?token=" + self.token, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local uri = self.uri; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("rev-gateway-secret") + .with_env_var("REV_GATEWAY_SECRET", "rev-gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "rev-gateway", + "-p", + url.pulsar, + "--websocket-uri", + std.toString(uri), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/structured-data.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/structured-data.jsonnet new file mode 100644 index 00000000..8b23372f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/structured-data.jsonnet @@ -0,0 +1,171 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "nlp-query" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("nlp-query") + .with_image(images.trustgraph_flow) + .with_command([ + "nlp-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "nlp-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-query" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("structured-query") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "structured-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-diag" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "96M", + "memory-reservation":: "96M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("structured-diag") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-diag", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "structured-diag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-objects" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-objects", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/trustgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/trustgraph.jsonnet new file mode 100644 index 00000000..3cf12d26 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/core/trustgraph.jsonnet @@ -0,0 +1,473 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +local config_initialiser = import "configuration.jsonnet"; +local config = import "../runtime-config/trustgraph-config.jsonnet"; +local librarian = import "librarian.jsonnet"; +local mcp_server = import "mcp-server.jsonnet"; +local workbench = import "../ui/workbench-ui.jsonnet"; +local graphrag = import "graph-rag.jsonnet"; +local documentrag = import "document-rag.jsonnet"; +local prompt_template = import "prompt-template.jsonnet"; +local agent_manager = import "agent-manager-react.jsonnet"; +local structured_data = import "structured-data.jsonnet"; +local ddg = import "mcp/ddg-mcp-server.jsonnet"; + +// Helper to create a routing function for a target object +local route = function(target) + function(prefix, k, v) + local suffix = std.substr(k, std.length(prefix), std.length(k) - std.length(prefix)); + { [target] +: { [suffix]:: v } }; + +// Parameter prefix -> target object routing table +local routes = { + "prompt-rag-": route("prompt-rag"), + "prompt-": route("prompt"), + "text-completion-rag-": route("text-completion-rag"), + "text-completion-": route("text-completion"), + "embeddings-": route("embeddings"), + "api-gateway-": route("api-gateway"), + "chunk-": route("chunker"), + "graph-rag-": route("graph-rag"), + "graph-embeddings-": route("graph-embeddings"), + "kg-extract-definitions-": route("kg-extract-definitions"), + "kg-extract-relationships-": route("kg-extract-relationships"), + "kg-extract-agent-": route("kg-extract-agent"), + "kg-extract-ontology-": route("kg-extract-ontology"), + "kg-extract-objects-": route("kg-extract-objects"), + "garage-": route("garage"), + "config-svc-": route("config-svc"), + "pdf-decoder-": route("pdf-decoder"), + "mcp-tool-": route("mcp-tool"), + "mcp-server-": route("mcp-server"), + "metering-rag-": route("metering-rag"), + "metering-": route("metering"), + "kg-store-": route("kg-store"), + "kg-manager-": route("kg-manager"), + "librarian-": route("librarian"), + "agent-manager-": route("agent-manager"), + "document-rag-": route("document-rag"), + "document-embeddings-": route("document-embeddings"), + "rev-gateway-": route("rev-gateway"), + "nlp-query-": route("nlp-query"), + "structured-query-": route("structured-query"), + "structured-diag-": route("structured-diag"), + "init-trustgraph-": route("init-trustgraph"), +}; + +// Find longest matching prefix (most specific first) +local findRoute = function(k) + local prefixes = std.objectFields(routes); + local matching = std.filter(function(p) std.startsWith(k, p), prefixes); + local sorted = std.sort(matching, function(x) -std.length(x)); + if std.length(sorted) > 0 then sorted[0] else null; + +{ + + // Route parameters to appropriate internal objects based on prefix + with:: function(k, v) + local prefix = findRoute(k); + if prefix != null then + self + routes[prefix](prefix, k, v) + else + self + { [k]:: v }, + + "log-level":: "INFO", + + // Base objects with concurrency defaults (LLM/embeddings components merge into these) + "text-completion" +: { concurrency:: 1 }, + "text-completion-rag" +: { concurrency:: 1 }, + embeddings +: { concurrency:: 1 }, + + "api-gateway" +: { + + port:: 8088, + timeout:: 600, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local port = self.port; + local timeout = self.timeout; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("gateway-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "api-gateway", + "-p", + url.pulsar, + "--timeout", + std.toString(timeout), + "--port", + std.toString(port), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "chunker" +: { + + size:: 2000, + overlap:: 50, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local size = self.size; + local overlap = self.overlap; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + std.toString(size), + "--chunk-overlap", + std.toString(overlap), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "config-svc" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local cpuLimit = self["cpu-limit"]; + local cpuReservation = self["cpu-reservation"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("config-svc") + .with_image(images.trustgraph_flow) + .with_command([ + "config-svc", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(cpuLimit, memoryLimit) + .with_reservations(cpuReservation, memoryReservation); + + local containerSet = engine.containers( + "config-svc", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "pdf-decoder" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("pdf-decoder") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-decoder", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "pdf-decoder", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "mcp-tool" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("mcp-tool") + .with_image(images.trustgraph_flow) + .with_command([ + "mcp-tool", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "mcp-tool", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("metering") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "metering", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering-rag" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("metering-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--id", + "metering-rag", + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "metering-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-store" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-store") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-store", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-store", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-manager", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + librarian + mcp_server + workbench + graphrag + + documentrag + prompt_template + agent_manager + structured_data + + config_initialiser + config + + ddg + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/embeddings/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/embeddings/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..37116de0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/embeddings/embeddings-fastembed.jsonnet @@ -0,0 +1,49 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-fastembed.jsonnet"; + +{ + + "fastembed-models":: models, + + "embeddings-models" +:: $["fastembed-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-fastembed", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/embeddings/embeddings-hf.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/embeddings/embeddings-hf.jsonnet new file mode 100644 index 00000000..492f05cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/embeddings/embeddings-hf.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-huggingface.jsonnet"; + +{ + + "huggingface-embeddings-models":: models, + + "embeddings-models" +:: $["huggingface-embeddings-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_hf) + .with_command([ + "embeddings-hf", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/embeddings/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/embeddings/embeddings-ollama.jsonnet new file mode 100644 index 00000000..05cbed72 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/embeddings/embeddings-ollama.jsonnet @@ -0,0 +1,52 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local models = import "parameters/embeddings-ollama.jsonnet"; + +{ + + "ollama-url":: "${OLLAMA_HOST}", + + "ollama-models":: models, + + "embeddings-models" +:: $["ollama-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-r", + $["ollama-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/aks-k8s.jsonnet new file mode 100644 index 00000000..07e050f2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/aks-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "disk.csi.azure.com", + parameters: { + // Standard disks (spinning magnetic), Locally Redundant Storage + // Cheapest, basically + skuName: "Standard_LRS", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/docker-compose.jsonnet new file mode 100644 index 00000000..334b3517 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/docker-compose.jsonnet @@ -0,0 +1,260 @@ +{ + + // Extract resources using the engine + package:: function(patterns) + std.foldl( + function(state, p) state + p.create(self), + std.objectValues(patterns), + {} + ), + + container:: function(name) + { + + local container = self, + + name:: name, + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_group:: function(x) self + + if std.objectHas(container, "group_add") then + { group_add: container.group_add + [x] } + else + { group_add: [x] }, + + with_command:: function(x) self + { + command: + if std.isString(x) then + std.strReplace(x, "$", "$$") + else if std.isArray(x) then + std.map(function(s) std.strReplace(s, "$", "$$"), x) + else + x + }, + + with_entrypoint:: function(x) self + { entrypoint: x }, + + with_runtime:: function(x) self + { runtime: x }, + + with_privileged:: function(x) self + { privileged: x }, + + with_ipc:: function(x) self + { ipc: x }, + + with_capability:: function(x) self + + if std.objectHas(container, "capability") then + { cap_add: container.capability + x } + else + { cap_add: [x], }, + + with_environment:: function(x) self + + if std.objectHas(container, "environment") then + { environment: container.environment + x } + else + { environment: x, }, + + with_device:: function(hdev, cdev) self + + if std.objectHas(container, "devices") then + { devices: container.devices + [ "%s:%s" % [hdev, cdev] ] } + else + { devices: [ "%s:%s" % [hdev, cdev] ], }, + + with_limits:: function(c, m) self + { + deploy +: { resources +: { + limits: { cpus: c, memory: m } + } }, + }, + + with_reservations:: function(c, m) self + { + deploy +: { resources +: { + reservations: { cpus: c, memory: m } + } }, + }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [vol.volid, mnt] + ] + else + [ + "%s:%s" % [vol.volid, mnt] + ] + }, + + with_bind_mount:: + function(src, dest) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [src, dest] + ] + else + [ + "%s:%s" % [src, dest] + ] + }, + + with_port:: + function(src, dest, name) + self + { + ports: + if std.objectHas(container, "ports") then + container.ports + [ "%d:%d" % [src, dest] ] + else + [ "%d:%d" % [src, dest] ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj.with_environment( + { [x]: "${" + x + "}" } + ), + vars.variables, + self + ), + + restart: "on-failure:100", + + add:: function() { + services +: { + [container.name]: container, + } + } + + }, + + internalService:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + with_size:: function(size) self + { size: size }, + + add:: function() { + volumes +: { + [volume.name]: {} + } + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: "./" + dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + variables:: [], + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + }, + + add:: function() { + } + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.foldl( + function(state, c) state + c.add(), + cont.containers, + {} + ), + + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/eks-k8s.jsonnet new file mode 100644 index 00000000..3fc3f035 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/eks-k8s.jsonnet @@ -0,0 +1,46 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "ebs.csi.aws.com", + parameters: { + type: "gp3", + encrypted: "true", + iops: "6000", + throughput: "400", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/gcp-k8s.jsonnet new file mode 100644 index 00000000..71792426 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/gcp-k8s.jsonnet @@ -0,0 +1,44 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "pd.csi.storage.gke.io", + parameters: { + type: "pd-balanced", + "csi.storage.k8s.io/fstype": "ext4", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/k8s.jsonnet new file mode 100644 index 00000000..2067f6ac --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/k8s.jsonnet @@ -0,0 +1,393 @@ +{ + + container:: function(name) + { + + local container = self, + + name: name, + limits: {}, + reservations: {}, + ports: [], + volumes: [], + bindMounts: [], + groups: [], + environment: [], + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_group:: function(x) self + { groups: super.groups + [x] }, + + with_privileged:: function(x) self + { privileged: x }, + + with_command:: function(x) self + { command: x }, + + with_entrypoint:: function(x) self + { entrypoint: x }, + + with_environment:: function(x) self + { + environment: super.environment + [ + { + name: v.key, value: v.value + } + for v in std.objectKeysValues(x) + ], + }, + + with_limits:: function(c, m) self + { limits: { cpu: c, memory: m } }, + + with_reservations:: + function(c, m) self + { reservations: { cpu: c, memory: m } }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: super.volumes + [{ + volume: vol, mount: mnt + }] + }, + + with_bind_mount:: + function(src, dest) + local name = "bind-" + std.strReplace(std.strReplace(src, "/", "-"), ".", "-"); + self + { + bindMounts: super.bindMounts + [{ + name: name, src: src, dest: dest + }] + }, + + with_port:: + function(src, dest, name) self + { + ports: super.ports + [ + { src: src, dest: dest, name : name } + ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj + { + environment: super.environment + [{ + name: x, + valueFrom: { + secretKeyRef: { + name: vars.name, + key: vars.keyMap[x], + } + } + }] + }, + vars.variables, + self + ), + + add:: function() [ + + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: container.name, + namespace: "trustgraph", + labels: { + app: container.name + } + }, + spec: { + replicas: 1, + selector: { + matchLabels: { + app: container.name, + } + }, + template: { + metadata: { + labels: { + app: container.name, + } + }, + spec: { + containers: [ + { + name: container.name, + image: container.image, + + // FIXME: Make everything run as + // root. Needed to get filesystems + // to be accessible. There's a + // better way of doing this? + securityContext: { + runAsUser: 0, + runAsGroup: 0, + } + ( + if std.objectHas(container, "privileged") && container.privileged then + { privileged: true } + else {} + ), + + resources: { + requests: container.reservations, + limits: container.limits + }, + } + ( + if std.length(container.ports) > 0 then + { + ports: [ + { + hostPort: port.src, + containerPort: port.dest, + } + for port in container.ports + ] + } else + {}) + + + (if std.objectHas(container, "entrypoint") then + // Entrypoint is set - use command for entrypoint, args for command + (if std.isString(container.entrypoint) && container.entrypoint == "" then + { command: [] } + else if std.isArray(container.entrypoint) then + { command: container.entrypoint } + else + { command: [container.entrypoint] } + ) + (if std.objectHas(container, "command") then + { args: container.command } + else {}) + else if std.objectHas(container, "command") then + { command: container.command } + else {}) + + + (if std.length(container.environment) > 0 then + { + env: container.environment, + } + else {}) + + + (if std.length(container.volumes) > 0 || std.length(container.bindMounts) > 0 then + { + volumeMounts: [ + { + mountPath: vol.mount, + name: vol.volume.name, + } + for vol in container.volumes + ] + [ + { + mountPath: bm.dest, + name: bm.name, + } + for bm in container.bindMounts + ] + } + + else + {} + ) + ], + volumes: [ + vol.volume.volRef() + for vol in container.volumes + ] + [ + { + name: bm.name, + hostPath: { path: bm.src } + } + for bm in container.bindMounts + ] + } + ( + if std.length(container.groups) > 0 then + { securityContext: { supplementalGroups: container.groups } } + else {} + ) + }, + } + {} + + } + + ] + + }, + + // Just an alias + internalService:: self.service, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + ports: [], + + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + + add:: function() [ + + { + + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + storageClassName: "tg", + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + data: parts + }, + ], + + + volRef:: function() { + name: volume.name, + configMap: { name: volume.name }, + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + variables: [], + keyMap: {}, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + }, + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + keyMap: super.keyMap + { [name]: key }, + }, + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.flattenArrays( + [ c.add() for c in cont.containers ] + ), + + }, + + resources:: function(res) + + std.flattenArrays( + [ c.add() for c in res ] + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/minikube-k8s.jsonnet new file mode 100644 index 00000000..858b17ad --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/minikube-k8s.jsonnet @@ -0,0 +1,115 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList, + + volume:: function(name) + { + local volume = self, + name: name, + with_size:: function(size) self + { size: size }, + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolume", + metadata: { + name: volume.name, + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + capacity: { + storage: volume.size, + }, + persistentVolumeReclaimPolicy: "Delete", + hostPath: { + path: "/data/pv-" + volume.name, + }, + } + }, + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + service:: function(containers) + { + local service = self, + name: containers.name, + ports: [], + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + add:: function() [ + { + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + type: "LoadBalancer", + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/noop.jsonnet new file mode 100644 index 00000000..1f384648 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/noop.jsonnet @@ -0,0 +1,79 @@ +{ + + // Extract resources usnig the engine + package:: function(patterns) {}, + + container:: function(name) { + + with_image:: function(x) self + {}, + + with_user:: function(x) self + {}, + + with_command:: function(x) self + {}, + + with_runtime:: function(x) self + {}, + + with_privileged:: function(x) self + {}, + + with_ipc:: function(x) self + {}, + + with_capability:: function(x) self + {}, + + with_environment:: function(x) self + {}, + + with_device:: function(hdev, cdev) self + {}, + + with_limits:: function(c, m) self + {}, + + with_reservations:: function(c, m) self + {}, + + with_volume_mount:: self + {}, + + with_port:: function(src, dest, name) self + {}, + + with_env_var_secrets:: function(vars) self + {}, + + add:: function() {}, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + service:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + volume:: function(name) { + with_size:: function(size) self + {}, + add:: function() {}, + }, + + configVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + secretVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self + {}, + add:: function() {}, + }, + + containers:: function(name, containers) { + add:: function() {}, + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/ovh-k8s.jsonnet new file mode 100644 index 00000000..15d57c83 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/ovh-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "cinder.csi.openstack.org", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", + parameters: { + availability: "nova", + fsType: "ext4", + type: "high-speed", + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/scw-k8s.jsonnet new file mode 100644 index 00000000..eed23968 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/engine/scw-k8s.jsonnet @@ -0,0 +1,40 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "csi.scaleway.com", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/agent-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/agent-extract.jsonnet new file mode 100644 index 00000000..4c5785c8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/agent-extract.jsonnet @@ -0,0 +1,35 @@ +// Agent-based extraction module +// Uses AI agents for more sophisticated knowledge extraction from text +// Leverages agent tools and reasoning for complex extraction tasks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - internal agent extraction service + "interfaces" +: { + }, + + // No configurable parameters for agent extraction + "parameters" +: { + }, + + // Flow-level processors for agent-based extraction + "flow" +: { + // Agent-based knowledge extraction processor + // Uses AI agents with tools to extract structured knowledge + "kg-extract-agent:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output knowledge triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "agent-request": request("agent:{id}"), // Agent service requests + "agent-response": response("agent:{id}"), // Agent service responses + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/agent.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/agent.jsonnet new file mode 100644 index 00000000..fbb06695 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/agent.jsonnet @@ -0,0 +1,59 @@ +// Agent management module +// Provides AI agent orchestration and tool integration +// Manages agent conversations, tool calls, and response coordination +// Supports MCP tools, GraphRAG, and structured queries + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // External interfaces for agent operations + "interfaces" +: { + "agent": request_response("agent:{id}"), // Main agent service interface + "mcp-tool": request_response("mcp-tool:{id}"), // MCP tool execution interface + }, + + // No configurable parameters for agent management + "parameters" +: { + }, + + // Flow-level processors for agent management + "flow" +: { + // Agent manager orchestrates agent conversations and tool usage + "agent-manager:{id}": { + // Agent communication channels + request: request("agent:{id}"), // Incoming agent requests + next: request("agent:{id}"), // Multi-turn conversation support + response: response("agent:{id}"), // Agent responses + + // LLM and prompt services + "text-completion-request": request("text-completion:{id}"), // LLM requests + "text-completion-response": response("text-completion:{id}"), // LLM responses + "prompt-request": request("prompt:{id}"), // Prompt processing + "prompt-response": response("prompt:{id}"), + + // Tool integrations + "mcp-tool-request": request("mcp-tool:{id}"), // MCP tool calls + "mcp-tool-response": response("mcp-tool:{id}"), + "graph-rag-request": request("graph-rag:{id}"), // GraphRAG queries + "graph-rag-response": response("graph-rag:{id}"), + "structured-query-request": request("structured-query:{id}"), // Structured data queries + "structured-query-response": response("structured-query:{id}"), + }, + + // MCP tool executor for agent tool usage + "mcp-tool:{id}": { + request: request("mcp-tool:{id}"), // Tool invocation requests + response: response("mcp-tool:{id}"), // Tool execution results + "text-completion-request": request("text-completion:{id}"), // LLM for tool reasoning + "text-completion-response": response("text-completion:{id}"), + }, + + }, + + // Blueprint-level processors for agent-related services + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/documentrag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/documentrag.jsonnet new file mode 100644 index 00000000..32597c79 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/documentrag.jsonnet @@ -0,0 +1,100 @@ +// Document RAG (Retrieval Augmented Generation) module +// Implements document-based RAG using chunk embeddings +// Provides semantic search and context-aware question answering +// Supports MCP (Model Context Protocol) tool integration + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces for document RAG functionality + "interfaces" +: { + // Document embedding storage and retrieval + "document-embeddings-store": flow("document-embeddings-store:{id}"), // Embedding storage stream + "document-rag": request_response("document-rag:{id}"), // Main document RAG interface + "document-embeddings": request_response("document-embeddings:{id}"), // Document embedding queries + + // Supporting services + "embeddings": request_response("embeddings:{id}"), // General embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing + "mcp-tool": request_response("mcp-tool:{id}"), // MCP tool integration + "text-completion": request_response("text-completion:{id}"), // LLM text completion + }, + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors for document embedding and storage + "flow" +: { + "document-embeddings:{id}": { + input: flow("chunk-load:{id}"), + output: flow("document-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "de-write:{id}": { + input: flow("document-embeddings-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "document-rag:{id}": { + request: request("document-rag:{id}"), + response: response("document-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "document-embeddings-request": request("document-embeddings:{id}"), + "document-embeddings-response": response("document-embeddings:{id}"), + }, + "de-query:{id}": { + request: request("document-embeddings:{id}"), + response: response("document-embeddings:{id}"), + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "mcp-tool:{id}": { + request: request("mcp-tool:{id}"), + response: response("mcp-tool:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + // Blueprint-level processors for document RAG operations + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/flow-blueprints.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/flow-blueprints.jsonnet new file mode 100644 index 00000000..abeb11ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/flow-blueprints.jsonnet @@ -0,0 +1,105 @@ +// TrustGraph Flow Blueprints Configuration +// Defines different flow combinations for various use cases +// Each flow blueprint combines multiple functional modules to create complete processing pipelines +// +// Available modules: +// - graphrag: Graph-based RAG with knowledge graphs +// - documentrag: Document-based RAG with chunk embeddings +// - structured: Structured data processing and NLP queries +// - agent: AI agent orchestration and tool integration +// - load: Document loading and preprocessing +// - kg-base: Basic knowledge extraction from text +// - agent-extract: Agent-based knowledge extraction +// - kgcore: Knowledge graph core storage + +// Import all the modular flow components +local graphrag_part = import "graphrag.jsonnet"; +local kg_base_part = import "kg-base.jsonnet"; +local onto_base_part = import "onto-base.jsonnet"; +local agent_extract_part = import "agent-extract.jsonnet"; +local structured_part = import "structured.jsonnet"; +local documentrag_part = import "documentrag.jsonnet"; +local agent_part = import "agent.jsonnet"; +local load_part = import "load.jsonnet"; +local kgcore_part = import "kgcore.jsonnet"; + +{ + + // Complete TrustGraph system with all capabilities + // Includes GraphRAG, DocumentRAG, structured data processing, and knowledge cores + "everything": { + description: "GraphRAG, DocumentRAG, structured data + knowledge cores", + tags: [ + "document-rag", "graph-rag", "knowledge-extraction", + "structured-data", "kgcore" + ], + } + + graphrag_part + documentrag_part + agent_part + load_part + + kg_base_part + structured_part, + + // Dual RAG system without knowledge core creation + // Combines both document and graph-based retrieval + "document-rag+graph-rag": { + description: "Supports GraphRAG and document RAG, no core creation", + tags: ["document-rag", "graph-rag", "knowledge-extraction"], + } + + graphrag_part + documentrag_part + agent_part + load_part + kg_base_part, + + // Graph-based RAG only + // Uses knowledge graphs for context-aware question answering + "graph-rag": { + description: "GraphRAG only", + tags: ["graph-rag", "knowledge-extraction"], + } + + graphrag_part + agent_part + load_part + kg_base_part, + + // Graph-based RAG only + // Uses knowledge graphs for context-aware question answering + "onto-rag": { + description: "Ontology RAG only", + tags: ["graph-rag", "knowledge-extraction"], + } + + graphrag_part + agent_part + load_part + onto_base_part, + + // Document-based RAG only + // Uses document embeddings for semantic search and answers + "document-rag": { + description: "DocumentRAG only", + tags: ["document-rag"], + } + + documentrag_part + load_part, + + // Full RAG system with knowledge core creation + // Includes both RAG types plus persistent knowledge storage + "document-rag+graph-rag+kgcore": { + description: "GraphRAG + DocumentRAG + knowledge core creation", + tags: ["document-rag", "graph-rag", "knowledge-extraction"], + } + + graphrag_part + documentrag_part + agent_part + load_part + + kgcore_part + kg_base_part, + + // GraphRAG with advanced agent-based extraction + // Uses AI agents for sophisticated knowledge extraction + "graph-rag+agent-extract": { + description: "GraphRAG + agent extract", + tags: ["graph-rag", "knowledge-extraction", "agent-extract"], + } + + graphrag_part + agent_part + load_part + agent_extract_part, + + // GraphRAG with structured data processing + // Combines knowledge graphs with structured data queries + "graph-rag+structured-data": { + description: "GraphRAG + structured data", + tags: ["graph-rag", "knowledge-extraction", "structured-data"], + } + + graphrag_part + agent_part + load_part + structured_part, + + // Structured data processing only + // Handles structured data extraction and NLP queries + "structured-data": { + description: "Structured data only", + tags: ["knowledge-extraction", "structured-data"], + } + + agent_part + load_part + structured_part, + +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/graphrag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/graphrag.jsonnet new file mode 100644 index 00000000..32ce5cf0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/graphrag.jsonnet @@ -0,0 +1,107 @@ +// GraphRAG flow configuration module +// Implements graph-based retrieval augmented generation (GraphRAG) functionality +// Handles knowledge graph storage, embeddings, and graph-based question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces exposed by the GraphRAG flow + "interfaces" +: { + // Data ingestion interfaces for graph construction + "entity-contexts-load": flow("entity-contexts-load:{id}"), // Entity context data stream + "triples-store": flow("triples-store:{id}"), // RDF triples storage stream + "graph-embeddings-store": flow("graph-embeddings-store:{id}"), // Graph embedding storage + + // Query interfaces for graph-based operations + "graph-rag": request_response("graph-rag:{id}"), // Main GraphRAG query interface + "triples": request_response("triples:{id}"), // Triple store queries + "graph-embeddings": request_response("graph-embeddings:{id}"), // Graph embedding queries + + // Supporting services + "embeddings": request_response("embeddings:{id}"), // General embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing service + "text-completion": request_response("text-completion:{id}"), // LLM text completion + }, + + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors - handle data streams for a specific flow instance + "flow" +: { + "graph-embeddings:{id}": { + input: flow("entity-contexts-load:{id}"), + output: flow("graph-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "triples-write:{id}": { + input: flow("triples-store:{id}"), + }, + "ge-write:{id}": { + input: flow("graph-embeddings-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "graph-rag:{id}": { + request: request("graph-rag:{id}"), + response: response("graph-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "graph-embeddings-request": request("graph-embeddings:{id}"), + "graph-embeddings-response": response("graph-embeddings:{id}"), + "triples-request": request("triples:{id}"), + "triples-response": response("triples:{id}"), + }, + "triples-query:{id}": { + request: request("triples:{id}"), + response: response("triples:{id}"), + }, + "ge-query:{id}": { + request: request("graph-embeddings:{id}"), + response: response("graph-embeddings:{id}"), + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + // Blueprint-level processors - shared across all flow instances of this blueprint + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/helpers.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/helpers.jsonnet new file mode 100644 index 00000000..eabb3bf4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/helpers.jsonnet @@ -0,0 +1,29 @@ +// Helper functions for flow configuration +// Provides utility functions for constructing flow, request, and response URIs +// used throughout the TrustGraph flow configuration system + +// Creates a persistent flow URI for data streams +// Persistent flows retain messages until consumed +local flow(x) = "persistent://tg/flow/" + x; + +// Creates a non-persistent request URI for request-response patterns +// Non-persistent means messages are not retained if no consumer is present +local request(x) = "non-persistent://tg/request/" + x; + +// Creates a non-persistent response URI for request-response patterns +local response(x) = "non-persistent://tg/response/" + x; + +// Creates a request-response pair for bidirectional communication +// Returns an object with both request and response URIs +local request_response(x) = { + request: request(x), + response: response(x), +}; + +// Export all helper functions for use in other modules +{ + flow: flow, + request: request, + response: response, + request_response: request_response, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/kg-base.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/kg-base.jsonnet new file mode 100644 index 00000000..144513b7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/kg-base.jsonnet @@ -0,0 +1,44 @@ +// Knowledge Graph Base extraction module +// Provides basic knowledge extraction capabilities from text chunks +// Extracts entity definitions and relationships using prompt-based processing + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts entity definitions from text chunks + // Identifies and defines key entities mentioned in the text + "kg-extract-definitions:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output definition triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition extraction prompts + "prompt-response": response("prompt:{id}"), + }, + + // Extracts relationships between entities + // Identifies how entities are connected and interact + "kg-extract-relationships:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output relationship triples + "prompt-request": request("prompt:{id}"), // Relationship extraction prompts + "prompt-response": response("prompt:{id}"), + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/kgcore.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/kgcore.jsonnet new file mode 100644 index 00000000..bfc01ada --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/kgcore.jsonnet @@ -0,0 +1,31 @@ +// Knowledge Graph Core storage module +// Handles persistent storage of knowledge graph data +// Consolidates triples and graph embeddings into permanent storage +// Creates the core knowledge base for long-term use + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; + +{ + // No external interfaces - internal storage service + "interfaces" +: { + }, + + // No configurable parameters for core storage + "parameters" +: { + }, + + // Flow-level processors for knowledge graph storage + "flow" +: { + // Knowledge graph store consolidates extracted knowledge + // Takes processed triples and embeddings and stores them permanently + "kg-store:{id}": { + "triples-input": flow("triples-store:{id}"), // Input RDF triples stream + "graph-embeddings-input": flow("graph-embeddings-store:{id}"), // Input graph embeddings + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/llm-parameters.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/llm-parameters.jsonnet new file mode 100644 index 00000000..a99f72ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/llm-parameters.jsonnet @@ -0,0 +1,59 @@ +{ + + // LLM model selection for normal LLM + "llm-model": { + "type": "llm-model", + "description": "LLM model", + "order": 1, + "advanced": false, + }, + + // LLM model for RAG operations + "llm-rag-model": { + "type": "llm-model", + "description": "LLM model for RAG", + "order": 2, + "advanced": true, + "controlled-by": "llm-model", + }, + + // LLM model selection for normal LLM + "llm-temperature": { + "type": "llm-temperature", + "description": "LLM temperature", + "order": 3, + "advanced": true, + }, + + // LLM model selection for normal LLM + "llm-rag-temperature": { + "type": "llm-temperature", + "description": "LLM temperature for RAG", + "order": 4, + "advanced": true, + }, + + "embeddings-model": { + "type": "embeddings-model", + "description": "Embeddings model", + "order": 5, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-size": { + "type": "chunk-size", + "description": "Chunk size", + "order": 6, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-overlap": { + "type": "chunk-overlap", + "description": "Chunk overlap", + "order": 7, + "advanced": true, + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/load.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/load.jsonnet new file mode 100644 index 00000000..00328b7f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/load.jsonnet @@ -0,0 +1,50 @@ +// Document loading and preprocessing module +// Handles document ingestion, format conversion, and chunking +// Converts PDFs to text and splits documents into processable chunks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + + // External interfaces for document loading + "interfaces" +: { + "document-load": flow("document-load:{id}"), // Raw document input stream + "text-load": flow("text-document-load:{id}"), // Text document stream + "embeddings": request_response("embeddings:{id}"), // Embedding service for chunks + }, + + // No configurable parameters for document loading + "parameters" +: { + }, + + // Flow-level processors for document preprocessing + "flow" +: { + // PDF decoder converts PDF documents to text + "pdf-decoder:{id}": { + input: flow("document-load:{id}"), // Raw PDF input + output: flow("text-document-load:{id}"), // Extracted text output + }, + + // Chunker splits documents into smaller, processable pieces + "chunker:{id}": { + input: flow("text-document-load:{id}"), // Full text documents + output: flow("chunk-load:{id}"), // Document chunks for processing + "chunk-size": "{chunk-size}", // Chunk size + "chunk-overlap": "{chunk-overlap}", // Overlap between chunks + }, + // Embedding service for converting text chunks to vectors + "embeddings:{id}": { + request: request("embeddings:{id}"), // Embedding requests + response: response("embeddings:{id}"), // Embedding responses + model: "{embeddings-model}", + }, + }, + + // Blueprint-level processors for document loading services + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/onto-base.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/onto-base.jsonnet new file mode 100644 index 00000000..d11daf64 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/onto-base.jsonnet @@ -0,0 +1,39 @@ +// Knowledge Graph Base extraction module +// Provides basic knowledge extraction capabilities from text chunks +// Extracts entity definitions and relationships using prompt-based processing + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts using ontology definitions + "kg-extract-ontology:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition + // extraction prompts + "prompt-response": response("prompt:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/ontorag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/ontorag.jsonnet new file mode 100644 index 00000000..32ce5cf0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/ontorag.jsonnet @@ -0,0 +1,107 @@ +// GraphRAG flow configuration module +// Implements graph-based retrieval augmented generation (GraphRAG) functionality +// Handles knowledge graph storage, embeddings, and graph-based question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces exposed by the GraphRAG flow + "interfaces" +: { + // Data ingestion interfaces for graph construction + "entity-contexts-load": flow("entity-contexts-load:{id}"), // Entity context data stream + "triples-store": flow("triples-store:{id}"), // RDF triples storage stream + "graph-embeddings-store": flow("graph-embeddings-store:{id}"), // Graph embedding storage + + // Query interfaces for graph-based operations + "graph-rag": request_response("graph-rag:{id}"), // Main GraphRAG query interface + "triples": request_response("triples:{id}"), // Triple store queries + "graph-embeddings": request_response("graph-embeddings:{id}"), // Graph embedding queries + + // Supporting services + "embeddings": request_response("embeddings:{id}"), // General embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing service + "text-completion": request_response("text-completion:{id}"), // LLM text completion + }, + + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors - handle data streams for a specific flow instance + "flow" +: { + "graph-embeddings:{id}": { + input: flow("entity-contexts-load:{id}"), + output: flow("graph-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "triples-write:{id}": { + input: flow("triples-store:{id}"), + }, + "ge-write:{id}": { + input: flow("graph-embeddings-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "graph-rag:{id}": { + request: request("graph-rag:{id}"), + response: response("graph-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "graph-embeddings-request": request("graph-embeddings:{id}"), + "graph-embeddings-response": response("graph-embeddings:{id}"), + "triples-request": request("triples:{id}"), + "triples-response": response("triples:{id}"), + }, + "triples-query:{id}": { + request: request("triples:{id}"), + response: response("triples:{id}"), + }, + "ge-query:{id}": { + request: request("graph-embeddings:{id}"), + response: response("graph-embeddings:{id}"), + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + // Blueprint-level processors - shared across all flow instances of this blueprint + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/structured.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/structured.jsonnet new file mode 100644 index 00000000..d56027ca --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/flows/structured.jsonnet @@ -0,0 +1,108 @@ +// Structured data processing module +// Handles extraction and querying of structured data objects +// Provides natural language to GraphQL query capabilities +// Supports structured data storage and retrieval + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // External interfaces for structured data operations + "interfaces" +: { + // Supporting services + "embeddings": request_response("embeddings:{id}"), // Embedding service + "prompt": request_response("prompt:{id}"), // Prompt processing + "text-completion": request_response("text-completion:{id}"), // LLM completion + + // Structured data storage and querying + "objects-store": flow("objects-store:{id}"), // Object storage stream + "objects": request_response("objects:{id}"), // Object query service + + // Query interfaces + "nlp-query": request_response("nlp-query:{id}"), // NLP to GraphQL translation + "structured-query": request_response("structured-query:{id}"), // Structured query execution + "structured-diag": request_response("structured-diag:{id}"), // Query diagnostics + }, + + + // Parameters that can be configured for this flow + "parameters" +: llm_parameters, + + // Flow-level processors for structured data extraction + "flow" +: { + "kg-extract-objects:{id}": { + input: flow("chunk-load:{id}"), + output: flow("objects-store:{id}"), + "entity-contexts": flow("entity-contexts-load:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + "objects-write:{id}": { + input: flow("objects-store:{id}"), + }, + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + "objects-query:{id}": { + request: request("objects:{id}"), + response: response("objects:{id}"), + }, + "nlp-query:{id}": { + request: request("nlp-query:{id}"), + response: response("nlp-query:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + }, + "structured-query:{id}": { + request: request("structured-query:{id}"), + response: response("structured-query:{id}"), + "nlp-query-request": request("nlp-query:{id}"), + "nlp-query-response": response("nlp-query:{id}"), + "objects-query-request": request("objects:{id}"), + "objects-query-response": response("objects:{id}"), + }, + "structured-diag:{id}": { + request: request("structured-diag:{id}"), + response: response("structured-diag:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + "metering:{id}": { + input: response("text-completion:{id}"), + }, + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + // Blueprint-level processors for structured data operations + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/azure-openai.jsonnet new file mode 100644 index 00000000..bee0fe47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/azure-openai.jsonnet @@ -0,0 +1,112 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure-openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-openai-" + key]:: value, + }, + +// Strategy is to specify the model with the AZURE_MODEL environment +// variable. This isn't something that can just be specified dynamically, +// it has to match what was provisioned in Azure. + + "azure-openai-max-output-tokens":: 4192, + "azure-openai-temperature":: 0.0, + "azure-openai-models":: models, + + "llm-models" +:: $["azure-openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/azure.jsonnet new file mode 100644 index 00000000..f4db7500 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/azure.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-" + key]:: value, + }, + + "azure-max-output-tokens":: 4096, + "azure-temperature":: 0.0, + "azure-models":: models, + + "llm-models" +:: $["azure-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/bedrock.jsonnet new file mode 100644 index 00000000..0db1475b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/bedrock.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/bedrock.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["bedrock-" + key]:: value, + }, + + "bedrock-max-output-tokens":: 4096, + "bedrock-temperature":: 0.0, + "bedrock-models":: models, + + "llm-models" +:: $["bedrock-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/claude.jsonnet new file mode 100644 index 00000000..f3125e96 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/claude.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/claude.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["claude-" + key]:: value, + }, + + "claude-max-output-tokens":: 4096, + "claude-temperature":: 0.0, + "claude-models":: models, + + "llm-models" +:: $["claude-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/cohere.jsonnet new file mode 100644 index 00000000..6517f7a0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/cohere.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/cohere.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["cohere-" + key]:: value, + }, + + "cohere-temperature":: 0.0, + "cohere-models":: models, + + "llm-models" +:: $["cohere-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/googleaistudio.jsonnet new file mode 100644 index 00000000..f7e8d578 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/googleaistudio.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/googleaistudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["googleaistudio-" + key]:: value, + }, + + "googleaistudio-max-output-tokens":: 4096, + "googleaistudio-temperature":: 0.0, + "googleaistudio-models":: models, + + "llm-models" +:: $["googleaistudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/llamafile.jsonnet new file mode 100644 index 00000000..f3bee2cc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/llamafile.jsonnet @@ -0,0 +1,94 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/slm.jsonnet"; +local models = import "parameters/llamafile.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["llamafile-" + key]:: value, + }, + + "llamafile-models":: models, + + "llm-models" +:: $["llamafile-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/lmstudio.jsonnet new file mode 100644 index 00000000..4cf78caa --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/lmstudio.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/lmstudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["lmstudio-" + key]:: value, + }, + + "lmstudio-max-output-tokens":: 4096, + "lmstudio-temperature":: 0.0, + "lmstudio-models":: models, + + "llm-models" +:: $["lmstudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/mistral.jsonnet new file mode 100644 index 00000000..2fb8c562 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/mistral.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/mistral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "mistral-max-output-tokens":: 4096, + "mistral-temperature":: 0.0, + "mistral-models":: models, + + "llm-models" +:: $["mistral-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/ollama.jsonnet new file mode 100644 index 00000000..6143f5cc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/ollama.jsonnet @@ -0,0 +1,102 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/ollama.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["ollama-" + key]:: value, + }, + + "ollama-models":: models, + + "llm-models" +:: $["ollama-models"], + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/openai.jsonnet new file mode 100644 index 00000000..c3d482fd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/openai.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["openai-" + key]:: value, + }, + + "openai-max-output-tokens":: 4096, + "openai-temperature":: 0.0, + "openai-models":: models, + + "llm-models" +:: $["openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/tgi.jsonnet new file mode 100644 index 00000000..91588dff --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/tgi.jsonnet @@ -0,0 +1,108 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-" + key]:: value, + }, + + "tgi-max-output-tokens":: 1024, + "tgi-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/vertexai.jsonnet new file mode 100644 index 00000000..cb8c141f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/vertexai.jsonnet @@ -0,0 +1,120 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vertexai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vertexai-" + key]:: value, + }, + + "vertexai-private-key":: "/vertexai/private.json", + "vertexai-region":: "us-central1", + "vertexai-max-output-tokens":: 4096, + "vertexai-temperature":: 0.0, + "vertexai-models":: models, + + "llm-models" +:: $["vertexai-models"], + + "text-completion" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion-rag") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/vllm.jsonnet new file mode 100644 index 00000000..8b846bbf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/llm/vllm.jsonnet @@ -0,0 +1,113 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vllm.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-" + key]:: value, + }, + + "vllm-models":: models, + + "llm-models" +:: $["vllm-models"], + + "vllm-max-output-tokens":: 1024, + "vllm-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/mcp/ddg-mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/mcp/ddg-mcp-server.jsonnet new file mode 100644 index 00000000..04176f15 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/mcp/ddg-mcp-server.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "ddg-mcp-server-port":: 9870, + + "ddg-mcp-server" +: { + + create:: function(engine) + + local port = $["ddg-mcp-server-port"]; + + local container = + engine.container("ddg-mcp-server") + .with_image(images["ddg-mcp-server"]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "ddg-mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + mcp +:: { + "duckduckgo": { + "remote-name": "search", + local port = $["ddg-mcp-server-port"], + local url = "http://ddg-mcp-server:%s/mcp" % port, + "url": url, + } + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/cpu-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/cpu-tgi.jsonnet new file mode 100644 index 00000000..04b61566 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/cpu-tgi.jsonnet @@ -0,0 +1,68 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + "tgi-service-storage":: "20G", + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-cpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--cuda-graphs", + "0", + ]) + .with_environment({ + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-battlemage-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-battlemage-vllm.jsonnet new file mode 100644 index 00000000..7646398c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-battlemage-vllm.jsonnet @@ -0,0 +1,98 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-Nemo-Instruct-2407", + "vllm-service-cpus":: "32.0", + "vllm-service-memory":: "48G", + "vllm-service-storage":: "48G", + "vllm-service-tokenizer-mode":: "mistral", + "vllm-service-datatype":: "float16", + "vllm-service-quantization":: "woq_int4", + "vllm-service-hf-token":: null, + "vllm-service-max-model-len":: 8192, + "vllm-service-max-num-seqs":: 16, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-battlemage"]) + .with_entrypoint("") // Clear default entrypoint + .with_command([ + "python", + "-m", + "ipex_llm.vllm.xpu.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--device", + "xpu", + "--dtype", + $["vllm-service-datatype"], + "--enforce-eager", + "--max-model-len", + std.toString($["vllm-service-max-model-len"]), + "--max-num-seqs", + std.toString($["vllm-service-max-num-seqs"]), + "--load-in-low-bit", + $["vllm-service-quantization"], + "--trust-remote-code", + "--tokenizer-mode", + $["vllm-service-tokenizer-mode"], + "--disable-sliding-window", + ]) + .with_environment({ + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS: "1", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-gaudi-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-gaudi-tgi.jsonnet new file mode 100644 index 00000000..5af36b78 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-gaudi-tgi.jsonnet @@ -0,0 +1,96 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "meta-llama/Llama-3.3-70B-Instruct", + "tgi-service-cpus":: "64.0", + "tgi-service-memory":: "64G", + "tgi-service-storage":: "50G", + "tgi-service-num-shard":: 8, + "tgi-service-max-input-tokens":: 4096, + "tgi-service-max-total-tokens":: 4096, + "tgi-service-max-batch-size":: 128, + "tgi-service-max-concurrent-requests":: 512, + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-gaudi"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--sharded", + "true", + "--num-shard", + std.toString($["tgi-service-num-shard"]), + "--max-input-tokens", + std.toString($["tgi-service-max-input-tokens"]), + "--max-total-tokens", + std.toString($["tgi-service-max-total-tokens"]), + "--max-batch-size", + std.toString($["tgi-service-max-batch-size"]), + "--max-waiting-tokens", + "7", + "--max-concurrent-requests", + std.toString($["tgi-service-max-concurrent-requests"]), + "--cuda-graphs", + "0", + ]) + .with_runtime("habana") + .with_environment({ + HABANA_VISIBLE_DEVICES: "all", + OMPI_MCA_btl_vader_single_copy_mechanism: "none", + ENABLE_HPU_GRAPH: "true", + LIMIT_HPU_GRAPH: "true", + USE_FLASH_ATTENTION: "true", + FLASH_ATTENTION_RECOMPUTE: "true", + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-gaudi-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-gaudi-vllm.jsonnet new file mode 100644 index 00000000..81d31c14 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-gaudi-vllm.jsonnet @@ -0,0 +1,76 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "64.0", + "vllm-service-memory":: "64G", + "vllm-service-storage":: "50G", + "vllm-service-tensor-parallel-size":: 8, + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-gaudi"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--tensor-parallel-size", + std.toString($["vllm-service-tensor-parallel-size"]), + ]) + .with_runtime("habana") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + HABANA_VISIBLE_DEVICES: "all", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-xpu-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-xpu-tgi.jsonnet new file mode 100644 index 00000000..bc806874 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-xpu-tgi.jsonnet @@ -0,0 +1,75 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + "tgi-service-storage":: "20G", + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-intel-xpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--cuda-graphs", + "0", + ]) + .with_environment({ + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_group("video") + .with_group("render") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-xpu-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-xpu-vllm.jsonnet new file mode 100644 index 00000000..f6e82905 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/intel-xpu-vllm.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "8.0", + "vllm-service-memory":: "16G", + "vllm-service-storage":: "20G", + "vllm-service-datatype":: "float16", + "vllm-service-max-model-len":: 4096, + "vllm-service-max-num-seqs":: 16, + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-xpu"]) + .with_command([ + "python", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--device", + "xpu", + "--dtype", + $["vllm-service-datatype"], + "--enforce-eager", + "--max-model-len", + std.toString($["vllm-service-max-model-len"]), + "--max-num-seqs", + std.toString($["vllm-service-max-num-seqs"]), + "--block-size", + "64", + "--gpu-memory-util", + "0.85", + "--trust-remote-code", + "--disable-sliding-window", + ]) + .with_environment({ + VLLM_USE_V1: "1", + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_group("video") + .with_group("render") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/nvidia-gpu-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/nvidia-gpu-vllm.jsonnet new file mode 100644 index 00000000..d04876c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/model-hosting/nvidia-gpu-vllm.jsonnet @@ -0,0 +1,72 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-7B-Instruct-v0.3", + "vllm-service-cpus":: "0.5", + "vllm-service-memory":: "1G", + "vllm-service-storage":: "50G", + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-nvidia"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + ]) + .with_runtime("nvidia") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/monitoring/grafana.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/monitoring/grafana.jsonnet new file mode 100644 index 00000000..dd0dddd7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/monitoring/grafana.jsonnet @@ -0,0 +1,124 @@ +local images = import "values/images.jsonnet"; +local loki = import "loki.jsonnet"; + +{ + + "prometheus" +: { + + create:: function(engine) + + local vol = engine.volume("prometheus-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "prometheus-cfg", "prometheus", + { + "prometheus.yml": importstr "prometheus/prometheus.yml", + } + ); + + local container = + engine.container("prometheus") + .with_image(images.prometheus) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M") + .with_port(9090, 9090, "http") + .with_volume_mount(cfgVol, "/etc/prometheus/") + .with_volume_mount(vol, "/prometheus"); + + local containerSet = engine.containers( + "prometheus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9090, 9090, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + + "grafana" +: { + + create:: function(engine) + + local vol = engine.volume("grafana-storage").with_size("20G"); + + local provDashVol = engine.configVolume( + "prov-dash", "grafana/provisioning/", + { + "dashboard.yml": + importstr "grafana/provisioning/dashboard.yml", + } + + ); + + local provDataVol = engine.configVolume( + "prov-data", "grafana/provisioning/", + { + "datasource.yml": + importstr "grafana/provisioning/datasource.yml", + } + + ); + + local dashVol = engine.configVolume( + "dashboards", "grafana/dashboards/", + { + "overview-dashboard.json": + importstr "grafana/dashboards/overview-dashboard.json", + "log-dashboard.json": + importstr "grafana/dashboards/log-dashboard.json", + } + + ); + + local container = + engine.container("grafana") + .with_image(images.grafana) + .with_environment({ + // GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin", + // GF_AUTH_ANONYMOUS_ENABLED: "true", + // GF_ORG_ROLE: "Admin", + GF_ORG_NAME: "trustgraph.ai", + // GF_SERVER_ROOT_URL: "https://example.com", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(3000, 3000, "cassandra") + .with_volume_mount(vol, "/var/lib/grafana") + .with_volume_mount( + provDashVol, "/etc/grafana/provisioning/dashboards/" + ) + .with_volume_mount( + provDataVol, "/etc/grafana/provisioning/datasources/" + ) + .with_volume_mount( + dashVol, "/var/lib/grafana/dashboards/" + ); + + local containerSet = engine.containers( + "grafana", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3000, 3000, "http"); + + engine.resources([ + vol, + provDashVol, + provDataVol, + dashVol, + containerSet, + service, + ]) + + }, + +} + loki + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/monitoring/loki.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/monitoring/loki.jsonnet new file mode 100644 index 00000000..0774f6cf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/monitoring/loki.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "loki" +: { + + create:: function(engine) + + local vol = engine.volume("loki-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "loki-cfg", "loki", + { + "local-config.yaml": importstr "loki/local-config.yaml", + } + ); + + local container = + engine.container("loki") + .with_image(images.loki) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(3100, 3100, "http") + .with_volume_mount(cfgVol, "/etc/loki/") + .with_volume_mount(vol, "/loki"); + + local containerSet = engine.containers( + "loki", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3100, 3100, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/ocr/mistral-ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/ocr/mistral-ocr.jsonnet new file mode 100644 index 00000000..f70e0cd6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/ocr/mistral-ocr.jsonnet @@ -0,0 +1,49 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "pdf-decoder" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("mistral-ocr") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-ocr-mistral", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "mistral-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/ocr/ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/ocr/ocr.jsonnet new file mode 100644 index 00000000..4f53aa5a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/ocr/ocr.jsonnet @@ -0,0 +1,37 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "pdf-decoder" +: { + + create:: function(engine) + + local container = + engine.container("pdf-ocr") + .with_image(images.trustgraph_ocr) + .with_command([ + "pdf-ocr", + "-p", + url.pulsar, + ]) + .with_limits("1.0", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "pdf-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/azure-openai.jsonnet new file mode 100644 index 00000000..0e3ea907 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/azure-openai.jsonnet @@ -0,0 +1,9 @@ +// Azure OpenAI LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-4o", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/azure.jsonnet new file mode 100644 index 00000000..6ffa5fdf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/azure.jsonnet @@ -0,0 +1,9 @@ +// Azure LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/bedrock.jsonnet new file mode 100644 index 00000000..df930e59 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/bedrock.jsonnet @@ -0,0 +1,63 @@ +// AWS Bedrock LLM Model Definitions +// Defines available models and their configurations for AWS Bedrock + +{ + "type": "string", + "description": "LLM model to use", + "default": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + "enum": [ + { + id: "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + description: "Claude Sonnet 4.5 (smartest for complex agents and coding)" + }, + { + id: "global.anthropic.claude-opus-4-5-20251101-v1:0", + description: "Claude Opus 4.5 (maximum intelligence)" + }, + { + id: "global.anthropic.claude-haiku-4-5-20251001-v1:0", + description: "Claude Haiku 4.5 (fastest with near-frontier intelligence)" + }, + { + id: "global.anthropic.claude-opus-4-1-20250805-v1:0", + description: "Claude Opus 4.1 (specialized reasoning)" + }, + { + id: "global.anthropic.claude-sonnet-4-20250514-v1:0", + description: "Claude Sonnet 4.0" + }, + { + id: "global.anthropic.claude-opus-4-20250514-v1:0", + description: "Claude Opus 4.0" + }, + { + id: "anthropic.claude-3-5-haiku-20241022-v1:0", + description: "Claude 3.5 Haiku" + }, + { + id: "anthropic.claude-3-haiku-20240307-v1:0", + description: "Claude 3 Haiku" + }, + { + id: "meta.llama3-1-405b-instruct-v1:0", + description: "Llama 3.1 405B Instruct" + }, + { + id: "meta.llama3-1-70b-instruct-v1:0", + description: "Llama 3.1 70B Instruct" + }, + { + id: "meta.llama3-1-8b-instruct-v1:0", + description: "Llama 3.1 8B Instruct" + }, + { + id: "mistral.mistral-large-2407-v1:0", + description: "Mistral Large" + }, + { + id: "mistral.mixtral-8x7b-instruct-v0:1", + description: "Mixtral 8x7B Instruct" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/chunking-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/chunking-param-types.jsonnet new file mode 100644 index 00000000..f82fa3f7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/chunking-param-types.jsonnet @@ -0,0 +1,25 @@ +// Chunk parameter type definitions + +{ + "chunk-size": { + "type": "integer", + "description": "Chunk size", + "placeholder": 2000, + "helper": "An integer, usually 2000 .. 8000", + "default": 2000, + "min": 0, + "max": 32768, + "required": true + }, + "chunk-overlap": { + "type": "integer", + "description": "Chunk overlap", + "placeholder": 50, + "helper": "An integer, usually 50 .. 100", + "default": 50, + "min": 0, + "max": 8000, + "required": true + }, +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/claude.jsonnet new file mode 100644 index 00000000..40fd434b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/claude.jsonnet @@ -0,0 +1,35 @@ +// Claude LLM Model Definitions +// Defines available models and their configurations for Anthropic's Claude + +{ + "type": "string", + "description": "LLM model to use", + "default": "claude-sonnet-4-5-20250929", + "enum": [ + { + id: "claude-sonnet-4-5-20250929", + description: "Claude Sonnet 4.5 (complex agents + coding)" + }, + { + id: "claude-opus-4-5-20251101", + description: "Claude Opus 4.5 (maximum intelligence)" + }, + { + id: "claude-haiku-4-5-20251001", + description: "Claude Haiku 4.5 (fast)" + }, + { + id: "claude-opus-4-1-20250805", + description: "Claude Opus 4.1 (specialized reasoning)" + }, + { + id: "claude-sonnet-4-20250514", + description: "Claude Sonnet 4.0" + }, + { + id: "claude-3-5-haiku-20241022", + description: "Claude 3.5 Haiku" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/cohere.jsonnet new file mode 100644 index 00000000..5a5865a3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/cohere.jsonnet @@ -0,0 +1,35 @@ +// Cohere LLM Model Definitions +// Defines available models and their configurations for Cohere + +{ + "type": "string", + "description": "LLM model to use", + "default": "command-r-plus-08-2024", + "enum": [ + { + id: "command-r-plus-08-2024", + description: "Command R+ (August 2024)" + }, + { + id: "command-r-08-2024", + description: "Command R (August 2024)" + }, + { + id: "command-r-plus", + description: "Command R+ (legacy)" + }, + { + id: "command-r", + description: "Command R (legacy)" + }, + { + id: "command", + description: "Command" + }, + { + id: "command-light", + description: "Command Light" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..477e25dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/embeddings-fastembed.jsonnet @@ -0,0 +1,127 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "sentence-transformers/all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "BAAI/bge-small-en-v1.5", + "description": "bge-small-en-v1.5" + }, + { + "id": "BAAI/bge-small-zh-v1.5", + "description": "bge-small-zh-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-xs", + "description": "snowflake-arctic-embed-xs" + }, + { + "id": "jinaai/jina-embeddings-v2-small-en", + "description": "jina-embeddings-v2-small-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "snowflake/snowflake-arctic-embed-s", + "description": "snowflake-arctic-embed-s" + }, + { + "id": "BAAI/bge-small-en", + "description": "bge-small-en" + }, + { + "id": "BAAI/bge-base-en-v1.5", + "description": "bge-base-en-v1.5" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "description": "paraphrase-multilingual-MiniLM-L12-v2" + }, + { + "id": "Qdrant/clip-ViT-B-32-text", + "description": "clip-ViT-B-32-text" + }, + { + "id": "jinaai/jina-embeddings-v2-base-de", + "description": "jina-embeddings-v2-base-de" + }, + { + "id": "BAAI/bge-base-en", + "description": "bge-base-en" + }, + { + "id": "snowflake/snowflake-arctic-embed-m", + "description": "snowflake-arctic-embed-m" + }, + { + "id": "thenlper/gte-base", + "description": "gte-base" + }, + { + "id": "jinaai/jina-embeddings-v2-base-en", + "description": "jina-embeddings-v2-base-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1", + "description": "nomic-embed-text-v1" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5", + "description": "nomic-embed-text-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-m-long", + "description": "snowflake-arctic-embed-m-long" + }, + { + "id": "jinaai/jina-clip-v1", + "description": "jina-clip-v1" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + { + "id": "jinaai/jina-embeddings-v2-base-es", + "description": "jina-embeddings-v2-base-es" + }, + { + "id": "jinaai/jina-embeddings-v2-base-code", + "description": "jina-embeddings-v2-base-code" + }, + { + "id": "jinaai/jina-embeddings-v2-base-zh", + "description": "jina-embeddings-v2-base-zh" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "description": "paraphrase-multilingual-mpnet-base-v2" + }, + { + "id": "snowflake/snowflake-arctic-embed-l", + "description": "snowflake-arctic-embed-l" + }, + { + "id": "BAAI/bge-large-en-v1.5", + "description": "bge-large-en-v1.5" + }, + { + "id": "thenlper/gte-large", + "description": "gte-large" + }, + { + "id": "intfloat/multilingual-e5-large", + "description": "multilingual-e5-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/embeddings-huggingface.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/embeddings-huggingface.jsonnet new file mode 100644 index 00000000..32254a2b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/embeddings-huggingface.jsonnet @@ -0,0 +1,31 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "all-mpnet-base-v2", + "description": "all-mpnet-base-v2" + }, + { + "id": "all-distilroberta-v1", + "description": "all-distilroberta-v1" + }, + { + "id": "stsb-bert-large", + "description": "stsb-bert-large" + }, + { + "id": "sentence-camembert-large", + "description": "sentence-camembert-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/embeddings-ollama.jsonnet new file mode 100644 index 00000000..c3f29fbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/embeddings-ollama.jsonnet @@ -0,0 +1,23 @@ +// Embeddings model definitions for Ollama +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "all-minilm:latest", + "enum": [ + { + "id": "all-minilm:latest", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/googleaistudio.jsonnet new file mode 100644 index 00000000..a8891932 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/googleaistudio.jsonnet @@ -0,0 +1,67 @@ +// Google AI Studio LLM Model Definitions +// Defines available models and their configurations for Google AI Studio + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 2.5 models (latest generation) + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-exp", + description: "Gemini 2.0 Flash (experimental)" + }, + + + // Claude models on VertexAI + { + id: "claude-3-5-sonnet@20241022", + description: "Claude 3.5 Sonnet (via VertexAI)" + }, + { + id: "claude-3-5-haiku@20241022", + description: "Claude 3.5 Haiku (via VertexAI)" + }, + { + id: "claude-3-opus@20240229", + description: "Claude 3 Opus (via VertexAI)" + }, + { + id: "claude-3-sonnet@20240229", + description: "Claude 3 Sonnet (via VertexAI)" + }, + { + id: "claude-3-haiku@20240307", + description: "Claude 3 Haiku (via VertexAI)" + }, + + // Llama models on VertexAI + { + id: "llama3-405b-instruct-maas", + description: "Llama 3 405B Instruct (via VertexAI)" + }, + { + id: "llama3-70b-instruct-maas", + description: "Llama 3 70B Instruct (via VertexAI)" + }, + { + id: "llama3-8b-instruct-maas", + description: "Llama 3 8B Instruct (via VertexAI)" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/llamafile.jsonnet new file mode 100644 index 00000000..f6480aa9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/llamafile.jsonnet @@ -0,0 +1,9 @@ +// LlamaFile LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/lmstudio.jsonnet new file mode 100644 index 00000000..1c88da3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/lmstudio.jsonnet @@ -0,0 +1,150 @@ +// LMStudio LLM Model Definitions +// Defines available models and their configurations for LMStudio + +{ + "type": "string", + "description": "LLM model to use", + "default": "llama3.1:70b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/mistral.jsonnet new file mode 100644 index 00000000..07f75b81 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/mistral.jsonnet @@ -0,0 +1,61 @@ +// Mistral LLM Model Definitions +// Defines available models and their configurations for Mistral AI + +{ + "type": "string", + "description": "LLM model to use", + "default": "mistral-medium-2508", + "enum": [ + // Featured models + { + id: "mistral-medium-2508", + description: "Mistral Medium 3.1" + }, + { + id: "mistral-large-2512", + description: "Mistral Large 3" + }, + { + id: "mistral-small-2506", + description: "Mistral Small 3.2" + }, + { + id: "ministral-14b-2512", + description: "Ministral 3 14B" + }, + { + id: "ministral-8b-2512", + description: "Ministral 3 8B" + }, + { + id: "ministral-3b-2512", + description: "Ministral 3 3B" + }, + { + id: "magistral-medium-2509", + description: "Magistral Medium 1.2 (reasoning)" + }, + { + id: "magistral-small-2509", + description: "Magistral Small 1.2 (reasoning)" + }, + { + id: "devstral-2512", + description: "Devstral 2 (code)" + }, + // Other models + { + id: "codestral-2508", + description: "Codestral (code)" + }, + { + id: "pixtral-large-2411", + description: "Pixtral Large (vision)" + }, + { + id: "open-mistral-nemo", + description: "Open Mistral Nemo" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/ollama.jsonnet new file mode 100644 index 00000000..4ef98a6a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/ollama.jsonnet @@ -0,0 +1,150 @@ +// Ollama LLM Model Definitions +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemma3:12b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/openai.jsonnet new file mode 100644 index 00000000..75b2670f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/openai.jsonnet @@ -0,0 +1,31 @@ +// OpenAI LLM Model Definitions +// Defines available models and their configurations for OpenAI's platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-4o", + "enum": [ + { + id: "gpt-4o", + description: "GPT-4o (latest)" + }, + { + id: "gpt-4o-mini", + description: "GPT-4o Mini" + }, + { + id: "gpt-4-turbo", + description: "GPT-4 Turbo" + }, + { + id: "gpt-4", + description: "GPT-4" + }, + { + id: "gpt-3.5-turbo", + description: "GPT-3.5 Turbo" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/temperature-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/temperature-param-types.jsonnet new file mode 100644 index 00000000..f5523e88 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/temperature-param-types.jsonnet @@ -0,0 +1,15 @@ +// LLM temperature definitions + +{ + "llm-temperature": { + "type": "float", + "description": "LLM temperature", + "placeholder": 0.3, + "helper": "A floating point number between 0 and 1", + "default": 0.3, + "min": 0.0, + "max": 10.0, + "required": true + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/vertexai.jsonnet new file mode 100644 index 00000000..6d90f8d3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/vertexai.jsonnet @@ -0,0 +1,68 @@ +// VertexAI LLM Model Definitions +// Defines available models and their configurations for Google's VertexAI platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 2.5 models (latest generation) + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-exp", + description: "Gemini 2.0 Flash (experimental)" + }, + + + // Claude models on VertexAI + { + id: "claude-3-5-sonnet@20241022", + description: "Claude 3.5 Sonnet (via VertexAI)" + }, + { + id: "claude-3-5-haiku@20241022", + description: "Claude 3.5 Haiku (via VertexAI)" + }, + { + id: "claude-3-opus@20240229", + description: "Claude 3 Opus (via VertexAI)" + }, + { + id: "claude-3-sonnet@20240229", + description: "Claude 3 Sonnet (via VertexAI)" + }, + { + id: "claude-3-haiku@20240307", + description: "Claude 3 Haiku (via VertexAI)" + }, + + // Llama models on VertexAI + { + id: "llama3-405b-instruct-maas", + description: "Llama 3 405B Instruct (via VertexAI)" + }, + { + id: "llama3-70b-instruct-maas", + description: "Llama 3 70B Instruct (via VertexAI)" + }, + { + id: "llama3-8b-instruct-maas", + description: "Llama 3 8B Instruct (via VertexAI)" + }, + ], + "required": true +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/vllm.jsonnet new file mode 100644 index 00000000..120342b2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/parameters/vllm.jsonnet @@ -0,0 +1,18 @@ +// vLLM Model Definitions +// Defines available models and their configurations for vLLM +// vLLM works with any HuggingFace model. We have to use the model +// vLLM was initialised with + +{ + "type": "string", + "description": "LLM model to use", + "default": "model", + "enum": [ + // Llama 3.1 models + { + id: "model", + description: "Pre-defined model" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/profiles/memory-profile-low.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/profiles/memory-profile-low.jsonnet new file mode 100644 index 00000000..e6341062 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/profiles/memory-profile-low.jsonnet @@ -0,0 +1,169 @@ +// Low memory profile - reduces memory allocation across components. +// Include this at the END of your configuration to override default memory +// settings with lower values suitable for memory-constrained environments. +// +// Usage: {"name": "memory-profile-low", "parameters": {}} +// +// Note: This trades some performance headroom for reduced memory usage. +// Monitor for OOM errors under heavy load. + +{ + + // Override Pulsar stack memory settings + "pulsar" +: { + + // Zookeeper: 512M -> 300M + "zk-memory-limit":: "300M", + "zk-memory-reservation":: "200M", + "zk-heap":: "128m", + "zk-direct-memory":: "64m", + + // Bookie: 1024M -> 600M + "bookie-memory-limit":: "600M", + "bookie-memory-reservation":: "400M", + "bookie-heap":: "128m", + "bookie-direct-memory":: "128m", + + // Broker: 800M -> 512M + "broker-memory-limit":: "512M", + "broker-memory-reservation":: "400M", + "broker-heap":: "192m", + "broker-direct-memory":: "192m", + + // Pulsar-init: 256M -> 128M + "init-memory-limit":: "128M", + "init-memory-reservation":: "128M", + "init-heap":: "64m", + "init-direct-memory":: "64m", + + }, + + // Override Cassandra memory settings: 1000M -> 600M + "cassandra" +: { + "memory-limit":: "600M", + "memory-reservation":: "500M", + "heap":: "200M", + }, + + // Override Qdrant memory settings: 1024M -> 600M + // Also enables mmap for vectors/payloads (trades latency for memory) + "qdrant" +: { + "memory-limit":: "600M", + "memory-reservation":: "500M", + "memmap-threshold-kb":: "1", + "on-disk-payload":: "true", + }, + + // TrustGraph core services - 50% memory reservations + "api-gateway" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "chunker" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "config-svc" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "pdf-decoder" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "mcp-tool" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "mcp-server" +: { + "memory-reservation":: "128M", // 256M -> 128M + }, + + "metering" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "metering-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-store" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-manager" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "prompt" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "prompt-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-embeddings" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "librarian" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "agent-manager" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + // Graph RAG services + "kg-extract-definitions" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-relationships" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-agent" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-ontology" +: { + "memory-reservation":: "150M", // 300M -> 150M + }, + + "kg-extract-objects" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "graph-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "graph-embeddings" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + // Structured data services + "nlp-query" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "structured-query" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "structured-diag" +: { + "memory-reservation":: "48M", // 96M -> 48M + }, + + // Init service + "init-trustgraph" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/agent-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/agent-prompt.txt new file mode 100644 index 00000000..9155d20f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/agent-prompt.txt @@ -0,0 +1,104 @@ +# ReAct Agent System Prompt + +You are an AI assistant that uses the ReAct (Reasoning + Acting) framework to solve problems through systematic reasoning and tool use. + +## Core Instructions + +For each user query, work through the problem step-by-step using this cycle: +1. **Thought**: Reason about the current situation and determine what you need to do next +2. **Action**: Take ONE specific action using an available tool +3. Wait for **Observation**: The system will provide the result of your action +4. Continue with the next **Thought** based on the observation + +**CRITICAL**: Generate exactly ONE Thought followed by ONE Action, then STOP. Do not generate multiple Thought/Action pairs in a single response. Do not generate Observations yourself - the system will provide them. + +## Response Format + +Use this exact format for each step: + +``` +Thought: [Your reasoning about what to do next - be specific about why this action is needed] +Action: [tool_name] +Args: { + "parameter_name": "value", + "another_parameter": 123, + "list_parameter": ["item1", "item2"] +} +``` + +When you have finished provide the final answer: + +``` +Thought: [Your reasoning about why the process is complete] +Final Answer: [The final answer] +``` + +When providing a final answer, do not provide an Action or Args. + +## Action Format Rules + +1. **Tool Name**: Write "Action: " followed by the exact tool name on its own line +2. **Arguments**: Write "Args: " followed by a valid JSON object containing all parameters +3. **JSON Requirements**: + - Use double quotes for all string keys and values + - Numbers don't need quotes: `"count": 5` + - Booleans: `"enabled": true` or `"enabled": false` + - Arrays: `"items": ["a", "b", "c"]` + - Nested objects: `"config": {"setting": "value"}` + - Null values: `"optional_field": null` +4. **Required Parameters**: Include all required parameters for the tool +5. **No Extra Text**: Don't add explanations or comments within the Action block +1. **Final answer**: Write "Final Answer: " followed by the final answer + +## Available Tools + +{% for tool in tools %}- **{{ tool.name }}**: {{ tool.description }} +{% for arg in tool.arguments %} - Required: `"{{ arg.name }}"` ({{ arg.type }}): {{ arg.description }} +{% endfor %} +{% endfor %} + +## Behavior Rules + +1. **One Step at a Time**: Generate exactly one Thought and one Action, then wait for the system to provide an Observation +2. **Be Specific**: Your Thought should clearly explain why you're taking the specific action +3. **Use Context**: Build on previous Observations to inform your next steps +4. **Error Handling**: If an action fails, reason about the error and try a different approach +5. **Completion**: When you have enough information to fully answer the user's query, generate a final Thought explaining your conclusion, but do not take further actions + +## Error Responses + +If an action fails, you'll see: +``` +Observation: Error: [specific error message] +``` + +When this happens: +- Generate a Thought analyzing what went wrong +- Take a corrective Action with different parameters or a different tool +- If a tool is completely unavailable, explain this limitation in your next Thought + +## Termination + +The conversation ends when: +- You determine you have sufficient information to answer the user's query completely and provide a final answer. +- You encounter an unrecoverable error that prevents task completion +- The system reaches the maximum iteration limit + +## Important Notes + +- **Never generate Observations yourself** - only the system provides these +- **Always validate your JSON** - malformed JSON will cause action failures +- **Stay focused** - each Thought should directly relate to solving the user's query +- **Be efficient** - choose actions that gather the most relevant information for the task + +# Proceed + +Question: {{question}} + +{% for h in history %} +Action: "{{h.action}}" +Args: { +{% for k, v in h.arguments.items() %} "{{k}}": "{{v}}" +{% endfor %}} +Observation: "{{h.observation}}" +{% endfor %} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/cohere.jsonnet new file mode 100644 index 00000000..9541e4c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/cohere.jsonnet @@ -0,0 +1,42 @@ +// For Cohere. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/default-prompts.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/default-prompts.jsonnet new file mode 100644 index 00000000..35dda682 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/default-prompts.jsonnet @@ -0,0 +1,234 @@ + +// Prompt templates. For tidy JSONNET use, don't change these templates +// here, but use over-rides in the prompt directory + +{ + + "system-template":: "You are a helpful assistant.", + + "templates":: { + + "question":: { + "prompt": "{{question}}", + }, + + "extract-definitions":: { + "prompt": "\nStudy the following text and derive definitions for any discovered entities.\nDo not provide definitions for entities whose definitions are incomplete\nor unknown.\nOutput relationships in JSON format as an array of objects with fields:\n- entity: the name of the entity\n- definition: English text which defines the entity\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract will be written as plain text. Do not add markdown formatting\nor headers or prefixes. Do not include null or unknown definitions.\n", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "entity": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "entity", + "definition" + ] + } + } + }, + + "extract-relationships":: { + "prompt": "\nStudy the following text and derive entity relationships. For each\nrelationship, derive the subject, predicate and object of the relationship.\nOutput relationships in JSON format as an array of objects with fields:\n- subject: the subject of the relationship\n- predicate: the predicate\n- object: the object of the relationship\n- object-entity: false if the object is a simple data type: name, value or date. true if it is an entity.\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not use special characters in the abstract text. The\nabstract must be written as plain text. Do not add markdown formatting\nor headers or prefixes.\n", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "object": { + "type": "string" + }, + "object-entity": { + "type": "boolean" + }, + }, + "required": [ + "subject", + "predicate", + "object", + "object-entity" + ] + } + } + }, + + "extract-topics":: { + "prompt": "You are a helpful assistant that performs information extraction tasks for a provided text.\nRead the provided text. You will identify topics and their definitions in JSON.\n\nReading Instructions:\n- Ignore document formatting in the provided text.\n- Study the provided text carefully.\n\nHere is the text:\n{{text}}\n\nResponse Instructions: \n- Do not respond with special characters.\n- Return only topics that are concepts and unique to the provided text.\n- Respond only with well-formed JSON.\n- The JSON response shall be an array of objects with keys \"topic\" and \"definition\". \n- The JSON response shall use the following structure:\n\n```json\n[{\"topic\": string, \"definition\": string}]\n```\n\n- Do not write any additional text or explanations.", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "topic": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "topic", + "definition" + ] + } + } + }, + + "extract-rows":: { + "prompt": "\nStudy the following text and derive objects which match the schema provided.\n\nYou must output an array of JSON objects for each object you discover\nwhich matches the schema. For each object, output a JSON object whose fields\ncarry the name field specified in the schema.\n\n\n\n{{schema}}\n\n\n\n{{text}}\n\n\n\nYou will respond only with raw JSON format data. Do not provide\nexplanations. Do not add markdown formatting or headers or prefixes.\n", + "response-type": "json", + }, + + "kg-prompt":: { + "prompt": "Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere's the knowledge statements:\n{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}})\n{%endfor%}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "response-type": "text", + }, + + "document-prompt":: { + "prompt": "Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements.\n\nHere is the context:\n{{documents}}\n\nUse only the provided knowledge statements to respond to the following:\n{{query}}\n", + "response-type": "text", + }, + + "agent-react":: { + "prompt": importstr "agent-prompt.txt", + "response-type": "text" + }, + + "agent-kg-extract":: { + "prompt": "Analyze the following text and extract both entity definitions and relationships. Return the results as JSON with 'definitions' and 'relationships' arrays.\n\nFor definitions, extract entities and their explanations or descriptions.\nFor relationships, extract subject-predicate-object triples where subjects and objects are entities, and predicates are relationship types.\n\nText: {{text}}\n\nReturn JSON only, no other text. Use this exact format:\n{\n \"definitions\": [\n {\n \"entity\": \"entity_name\",\n \"definition\": \"definition_text\"\n }\n ],\n \"relationships\": [\n {\n \"subject\": \"subject_entity\",\n \"predicate\": \"relationship_type\",\n \"object\": \"object_entity_or_literal\",\n \"object-entity\": true\n }\n ]\n}\n", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "definitions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "entity": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "entity", + "definition" + ] + } + }, + "relationships": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "object": { + "type": "string" + }, + "object-entity": { + "type": "boolean" + } + }, + "required": [ + "subject", + "predicate", + "object" + ] + } + } + }, + "required": [ + "definitions", + "relationships" + ] + } + }, + + "schema-selection":: { + "prompt": importstr "schema-selection.txt", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "An array of schema names that are relevant to answering the given question" + } + }, + + "graphql-generation":: { + "prompt": importstr "graphql-generation.txt", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The GraphQL query string generated to answer the question" + }, + "variables": { + "type": "object", + "description": "Object containing any GraphQL variables needed for the query", + "additionalProperties": true + }, + "confidence": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "description": "Float between 0.0-1.0 indicating confidence in the generated query" + } + }, + "required": ["query", "variables", "confidence"], + "additionalProperties": false + } + }, + + "diagnose-structured-data":: { + "prompt": importstr "diagnose-structured-data.txt", + "response-type": "json", + }, + + "diagnose-xml":: { + "prompt": importstr "diagnose-xml.txt", + "response-type": "json", + }, + "diagnose-json":: { + "prompt": importstr "diagnose-json.txt", + "response-type": "json", + }, + "diagnose-csv":: { + "prompt": importstr "diagnose-csv.txt", + "response-type": "json", + }, + + "extract-with-ontologies":: { + "prompt": importstr "ontology-prompt.txt", + "response-type": "json", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-csv.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-csv.txt new file mode 100644 index 00000000..ad9725ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-csv.txt @@ -0,0 +1,370 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in CSV processing and delimiter-separated value formats. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured CSV data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## CSV Processing Expertise + +When working with CSV data, you must: + +1. **Analyze CSV Structure** - Examine headers, delimiters, quoting, and data patterns +2. **Identify Column Mappings** - Map source column names to target fields +3. **Handle Complex CSV Patterns** - Support various CSV formats including: + - Standard comma-separated values: `name,age,city` + - Alternative delimiters: tab-separated (TSV), pipe-separated, semicolon-separated + - Quoted fields with embedded delimiters: `"Last, First",25,"New York, NY"` + - Headers with spaces or special characters: `"Customer Name","Order Date","Total Amount"` + - Files with or without headers + - Multi-line fields with embedded newlines + +## CSV Format Configuration Guidelines + +For CSV format configurations, use these patterns: + +**Basic CSV Configuration:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "skip_rows": 0 + } + } +} +``` + +**Advanced CSV Options:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": "\t", // Tab-separated + "quote_char": "\"", + "escape_char": "\\", + "has_header": true, + "skip_rows": 2, // Skip metadata rows + "null_values": ["", "NULL", "N/A"], + "trim_whitespace": true, + "skip_blank_lines": true + } + } +} +``` + +**CRITICAL: Source Field Names in Mappings** + +When processing CSV files, the parser uses column headers (if present) or generates column indices as field names. Your source field names in mappings must match exactly: + +**CORRECT Example with Headers:** +CSV file: +```csv +Customer Name,Order Date,Total Amount,Status +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "Customer Name": "John Smith", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Customer Name", // βœ… Correct - matches header exactly + "source_field": "Order Date", // βœ… Correct - matches header exactly + "source_field": "Total Amount", // βœ… Correct - matches header exactly + "source_field": "Status" // βœ… Correct - matches header exactly +} +``` + +**CORRECT Example without Headers:** +CSV file without headers uses column indices: +```csv +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "0": "John Smith", + "1": "2024-01-15", + "2": "1000.50", + "3": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "0", // βœ… Correct - first column + "source_field": "1", // βœ… Correct - second column + "source_field": "2", // βœ… Correct - third column + "source_field": "3" // βœ… Correct - fourth column +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - Delimiter character (comma, tab, pipe, semicolon, etc.) + - Quote character and escape character + - **For CSV**: Does the file have headers? Sample structure + - Text encoding (UTF-8, Windows-1252, etc.) + - Any rows to skip (metadata, blank lines) + - How null/empty values are represented + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source column β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Date/time format conversions + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + - Row-level validation rules + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## CSV Structure Analysis + +When presented with CSV data, analyze: + +1. **Delimiter Detection**: What character separates the fields? +2. **Header Presence**: Does the first row contain column names? +3. **Quote Pattern**: Are fields quoted? What quote character is used? +4. **Data Types**: What types are present in each column? +5. **Null Representation**: How are empty/null values represented? +6. **Special Characters**: Are there embedded commas, quotes, or newlines? +7. **Encoding Issues**: Are there any character encoding problems? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + // CSV-specific parsing options + // delimiter, quote_char, has_header, skip_rows, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `split`, `strip_quotes` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_number`, `parse_currency` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `clean_whitespace`, `normalize_encoding` + +**Date/Time Operations:** +- `parse_date`, `format_date`, `date_component` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `numeric_range`, `date_range` + +## CSV-Specific Best Practices + +1. **Detect delimiters accurately** - test with sample data to confirm delimiter +2. **Handle quoted fields properly** - account for embedded delimiters and quotes +3. **Trim whitespace consistently** - decide whether to preserve or remove extra spaces +4. **Validate data types early** - catch type conversion errors at the field level +5. **Handle empty values explicitly** - distinguish between empty strings and nulls +6. **Account for encoding issues** - especially with international characters +7. **Validate row structure** - ensure consistent column counts across rows + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle CSV-specific edge cases** like malformed quotes, inconsistent delimiters +10. **Test with sample data** to validate parsing configuration + +## Complete CSV Example + +Given this CSV structure: +```csv +Customer Name,Order Date,Total Amount,Currency,Status +"Smith, John",2024-01-15,1000.50,USD,Active +"Doe, Jane",2024-01-16,750.25,EUR,Pending +"Johnson, Bob",2024-01-17,,USD,Cancelled +``` + +The parser will: +1. Use `delimiter: ","` and `quote_char: "\""` to parse fields correctly +2. Use `has_header: true` to treat first row as column names +3. Create this parsed data structure for each record: + ```json + { + "Customer Name": "Smith, John", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Currency": "USD", + "Status": "Active" + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "trim_whitespace": true, + "null_values": ["", "NULL"] + } + }, + "mappings": [ + { + "source_field": "Customer Name", // βœ… Matches header exactly + "target_field": "customer_name", + "transforms": [{"type": "trim"}] + }, + { + "source_field": "Order Date", // βœ… Matches header exactly + "target_field": "order_date", + "transforms": [{"type": "to_date", "format": "YYYY-MM-DD"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "Total Amount", // βœ… Matches header exactly + "target_field": "amount", + "transforms": [ + {"type": "to_float"}, + {"type": "default", "value": 0.0} + ] + }, + { + "source_field": "Currency", // βœ… Matches header exactly + "target_field": "currency_code", + "transforms": [{"type": "upper"}], + "validation": [{"type": "in_list", "values": ["USD", "EUR", "GBP"]}] + }, + { + "source_field": "Status", // βœ… Matches header exactly + "target_field": "order_status", + "transforms": [{"type": "lower"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the column headers exactly, or use column indices (0, 1, 2, etc.) for files without headers.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the CSV structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to delimiter detection, header identification, quoting patterns, and data type inference: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-json.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-json.txt new file mode 100644 index 00000000..2fe6341d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-json.txt @@ -0,0 +1,327 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in JSON processing and JSONPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured JSON data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## JSON Processing Expertise + +When working with JSON data, you must: + +1. **Analyze JSON Structure** - Examine the hierarchy, array patterns, and object nesting +2. **Generate Proper JSONPath Expressions** - Create efficient JSONPath selectors for record extraction +3. **Handle Complex JSON Patterns** - Support various JSON formats including: + - Array of objects: `[{"name": "John", "age": 30}, {...}]` + - Nested object arrays: `{"data": {"records": [{"id": 1}, {...}]}}` + - Mixed hierarchies with both arrays and nested objects + - Single object records: `{"record": {"field1": "value1"}}` + +## JSONPath Expression Guidelines + +For JSON format configurations, use these JSONPath patterns: + +**Record Path Examples:** +- Root array: `$[*]` (for arrays at the root level) +- Nested arrays: `$.data.records[*]` or `$.response.items[*]` +- Single object: `$.record` (when there's one record per file) +- Deep nesting: `$.data.results.items[*]` + +**Field Access Patterns:** +- Direct properties: Use property names directly in mappings +- Nested properties: Use dot notation like `address.street` or `contact.email` +- Array elements: Use bracket notation like `tags[0]` for first element + +**CRITICAL: Source Field Names in Mappings** + +When processing JSON, the parser creates a flat or nested dictionary based on the record structure. Your source field names in mappings must match the actual property names in the parsed records: + +**CORRECT Example:** +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "metadata": { + "source": "UN", + "year": 2024 + } +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches property name + "source_field": "Trade (USD)", // βœ… Correct - matches property name + "source_field": "metadata.source", // βœ… Correct - nested property access + "source_field": "metadata.year" // βœ… Correct - nested property access +} +``` + +**JSON Format Configuration Template:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$[*]", // JSONPath to extract records + "flatten_nested": true, // Whether to flatten nested objects + "array_handling": "expand" // How to handle arrays: expand, first, concat + } + } +} +``` + +**Alternative JSON Options:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.items[*]", // For nested array structures + "flatten_nested": false, // Keep nested structure + "null_value_handling": "skip" // skip, empty_string, or preserve + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - JSON structure type (array of objects, nested objects, single records) + - **For JSON**: Sample structure, nesting patterns, array locations + - Sample data or field descriptions + - Any format-specific details (encoding, special null handling, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Nested object flattening requirements + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or null values + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## JSON Structure Analysis + +When presented with JSON data, analyze: + +1. **Root Structure**: Is it an array, object, or nested structure? +2. **Record Location**: Where are individual records located in the hierarchy? +3. **Field Pattern**: How are field names and values structured? + - Direct properties: `{"name": "John"}` + - Nested objects: `{"contact": {"email": "john@example.com"}}` + - Arrays: `{"tags": ["red", "blue"]}` + - Mixed types: `{"data": [{"id": 1, "details": {"name": "John"}}]}` +4. **Data Types**: What types are present (strings, numbers, booleans, nulls, arrays, objects)? +5. **Hierarchy Depth**: How deeply nested are the records and fields? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + // JSON-specific parsing options + // record_path (JSONPath), flatten_nested, array_handling, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `flatten_object`, `extract_array_element`, `join_array` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## JSON-Specific Best Practices + +1. **Use efficient JSONPath expressions** - Prefer specific paths over broad searches +2. **Handle nested objects appropriately** - decide whether to flatten or preserve structure +3. **Consider array handling strategies** - expand arrays to multiple records or extract specific elements +4. **Account for null vs undefined** values in field mappings +5. **Handle mixed data types** within the same field across records +6. **Use appropriate flattening** for deeply nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle JSON-specific edge cases** like empty arrays, null objects, mixed types + +## Complete JSON Example + +Given this JSON structure: +```json +{ + "data": { + "records": [ + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata": { + "source": "World Bank", + "confidence": 0.95 + } + } + ] + } +} +``` + +The parser will: +1. Use `record_path: "$.data.records[*]"` to extract record objects from the array +2. Create this parsed data structure for each record: + ```json + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata.source": "World Bank", // If flattened + "metadata.confidence": 0.95 // If flattened + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.records[*]", + "flatten_nested": true, + "array_handling": "expand" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches property name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches property name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches property name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + }, + { + "source_field": "metadata.source", // βœ… Flattened nested field + "target_field": "data_source" + } + ] +} +``` + +**KEY RULE: source_field names must match the actual property names in the parsed JSON records, using dot notation for nested properties when flattened.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the JSON structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to JSON hierarchy, object patterns, array structures, and generate appropriate JSONPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-structured-data.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-structured-data.txt new file mode 100644 index 00000000..84c4b8be --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-structured-data.txt @@ -0,0 +1,309 @@ + +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` (will be converted to relative paths automatically) +- With namespaces: `//ns:record` or `//soap:Body/data/record` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name + +**CRITICAL: Source Field Names in Mappings** + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary. Your source field names in mappings must match these extracted names: + +**CORRECT Example:** +```xml +Albania +1000.50 +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50" +} +``` + +So your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)" // βœ… Correct - matches parsed field name +} +``` + +**INCORRECT Example:** +```json +{ + "source_field": "Field[@name='Country or Area']", // ❌ Wrong - XPath not needed here + "source_field": "field[@name='Trade (USD)']" // ❌ Wrong - XPath not needed here +} +``` + +**XML Format Configuration Template:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name" // For value pattern + } + } +} +``` + +**Alternative XML Options:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // Direct element-based records + // No field_attribute needed for standard XML + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - File type (CSV, JSON, XML, Excel, fixed-width, etc.) + - **For XML**: Sample structure, namespace prefixes, record element patterns + - Sample data or field descriptions + - Any format-specific details (delimiters, encoding, namespaces, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Record Container**: Where are individual records located? +3. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns +4. **Namespaces**: Are there any namespace prefixes? +5. **Hierarchy Depth**: How deeply nested are the records? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "[csv|json|xml|fixed-width|excel]", + "encoding": "utf-8", + "options": { + // Format-specific parsing options + // For XML: record_path (XPath), field_attribute (if applicable) + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches +2. **Handle namespace prefixes** when present +3. **Identify field attribute patterns** correctly +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes + +## Complete XML Example + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "/ROOT/data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the extracted field names, NOT the XML element structure.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, and generate appropriate XPath expressions: + +{{sample}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-xml.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-xml.txt new file mode 100644 index 00000000..a3d8efa4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/diagnose-xml.txt @@ -0,0 +1,453 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for XML data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured XML data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + - CDATA sections and text nodes + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` +- With namespaces: `//ns:record` or `//soap:Body/data/record` +- Attribute-filtered: `//record[@type='customer']` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name like `field_attribute: "id"` or `field_attribute: "type"` + +## CRITICAL: Source Field Names in Mappings + +The behavior depends on whether you use `field_attribute`: + +### With field_attribute (Attribute-Based Fields) + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary: + +**Example:** +```xml + + Albania + 1000.50 + Active + +``` + +Parser configuration: +```json +{ + "record_path": "//record", + "field_attribute": "name" +} +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)", // βœ… Correct - matches parsed field name + "source_field": "Status" // βœ… Correct - matches parsed field name +} +``` + +### Without field_attribute (Element-Based Fields) + +When NOT using `field_attribute`, use direct element names or XPath expressions: + +**Example:** +```xml + + Albania + 1000.50 + Active + + UN + 2024 + + +``` + +Parser configuration: +```json +{ + "record_path": "//record" + // No field_attribute specified +} +``` + +Your mappings should use: +```json +{ + "source_field": "country", // βœ… Direct element name + "source_field": "trade_amount", // βœ… Direct element name + "source_field": "metadata/source", // βœ… Nested element path + "source_field": "metadata/year" // βœ… Nested element path +} +``` + +## XML Format Configuration Templates + +**For Attribute-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name", // Extract field names from 'name' attribute + "namespace_prefixes": { // Optional: define namespaces + "ns": "http://example.com/namespace" + } + } + } +} +``` + +**For Element-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // XPath to find record elements + "preserve_namespaces": true, // Optional: keep namespace info + "ignore_attributes": false // Optional: include element attributes + } + } +} +``` + +**For Complex XML with Namespaces:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//soap:Body//data:record", + "namespace_prefixes": { + "soap": "http://schemas.xmlsoap.org/soap/envelope/", + "data": "http://example.com/data" + }, + "field_attribute": "name" + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **XML Structure Details** + - Sample XML structure or schema + - Root element and record location + - Field organization (elements vs attributes) + - Namespace declarations and prefixes + - Text encoding (UTF-8, ISO-8859-1, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Namespaces**: Are there namespace declarations? What prefixes are used? +3. **Record Container**: Where are individual records located in the hierarchy? +4. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns: `John` +5. **Data Location**: Are values in element text, attributes, or both? +6. **Hierarchy Depth**: How deeply nested are the records? +7. **Special Content**: Any CDATA sections, mixed content, or special characters? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + // XML-specific parsing options + // record_path (XPath), field_attribute (if applicable), namespace_prefixes + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `strip_cdata`, `decode_html_entities` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_xml_date`, `parse_iso_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `extract_attribute`, `get_text_content` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `valid_xml_name`, `namespace_check` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches like `//` when possible +2. **Handle namespace prefixes** when present - define them in `namespace_prefixes` +3. **Identify field attribute patterns** correctly - determine if using `field_attribute` +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures +7. **Handle CDATA sections** and special characters properly +8. **Preserve or normalize whitespace** as appropriate +9. **Consider XML schema validation** if XSD is available + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle XML-specific edge cases** like empty elements, mixed content, processing instructions + +## Complete XML Examples + +### Example 1: Attribute-Based Fields + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "//data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +### Example 2: Element-Based Fields + +Given this XML structure: +```xml + + + John Smith + john@example.com +
+ 123 Main St + New York +
+ + + 456 + 100.50 + + +
+
+``` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer" + } + }, + "mappings": [ + { + "source_field": "name", // βœ… Direct element name + "target_field": "customer_name" + }, + { + "source_field": "email", // βœ… Direct element name + "target_field": "email_address" + }, + { + "source_field": "address/street", // βœ… Nested element path + "target_field": "street_address" + }, + { + "source_field": "address/city", // βœ… Nested element path + "target_field": "city" + }, + { + "source_field": "@id", // βœ… Attribute reference + "target_field": "customer_id", + "transforms": [{"type": "to_int"}] + } + ] +} +``` + +**KEY RULES:** +- With `field_attribute`: source_field names must match the extracted attribute values +- Without `field_attribute`: use direct element names, nested paths, or XPath expressions +- Use `@attribute` notation for XML element attributes + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, namespace usage, and generate appropriate XPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/gemini.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/gemini.jsonnet new file mode 100644 index 00000000..b9a1e0c0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/gemini.jsonnet @@ -0,0 +1,42 @@ +// For VertexAI Gemini. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/graphql-generation.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/graphql-generation.txt new file mode 100644 index 00000000..58af61dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/graphql-generation.txt @@ -0,0 +1,59 @@ +You are a GraphQL query generation expert. Given a natural language question and provided database schemas, generate a valid GraphQL query embedded in valid JSON. + +## Question: +{{ question }} + +## Provided Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %}- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif %}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif %}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ field.enum_values|join(', ') }}]{% endif %} +{% endfor %} +{% endfor %} + +## GraphQL Query Rules: +1. Use the schema names as GraphQL query fields (e.g., `customers`, `orders`) +2. Apply filters using the `where` parameter with nested filter objects +3. Available filter operators per field type: + - String fields: `eq`, `contains`, `startsWith`, `endsWith`, `in`, `not`, `not_in` + - Integer/Float fields: `eq`, `gt`, `gte`, `lt`, `lte`, `in`, `not`, `not_in` +4. Use `order_by` for sorting (field name as string) +5. Use `direction` for sort direction: `ASC` or `DESC` +6. Use `limit` to restrict number of results +7. Select specific fields in the query body + +## Task Instructions: +1. Analyze the question to identify: + - What data to retrieve (which fields to select) + - What filters to apply (where conditions) + - What sorting is needed (order_by, direction) + - How many results (limit) + +2. Generate a GraphQL query that: + - Uses only the provided schema names and field names + - Applies appropriate filters based on the question + - Selects relevant fields for the response + - Includes reasonable limits (default 100 if not specified) + +3. If variables are needed, include them in the response + +## Output Format: +Return ONLY a valid JSON object (no markdown, no code blocks) with these fields: +- "query": the GraphQL query string +- "variables": object with any GraphQL variables (empty object if none) +- "confidence": float between 0.0-1.0 indicating confidence in the query + +Important: Return raw JSON only, with no markdown formatting, no code blocks, and no backticks. + +## Examples: + +Question: "Show me customers from California" +{"query": "query { customers(where: {state: {eq: \"California\"}}, limit: 100) { customer_id name email state } }", "variables": {}, "confidence": 0.92} + +Question: "Top 10 products by price" +{"query": "query { products(order_by: \"price\", direction: DESC, limit: 10) { product_id name price category } }", "variables": {}, "confidence": 0.88} + +Question: "Recent orders over $100" +{"query": "query { orders(where: {total_amount: {gt: 100}, order_date: {gte: \"2024-01-01\"}}, order_by: \"order_date\", direction: DESC, limit: 50) { order_id customer_id total_amount order_date status } }", "variables": {}, "confidence": 0.96} + +Now generate the GraphQL query for the question above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/mixtral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/mixtral.jsonnet new file mode 100644 index 00000000..cd56e7ef --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/mixtral.jsonnet @@ -0,0 +1,42 @@ +// For Mixtral. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/ontology-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/ontology-prompt.txt new file mode 100644 index 00000000..5a0fcce0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/ontology-prompt.txt @@ -0,0 +1,126 @@ +You are a knowledge extraction expert. Your task is to find entities, relationships, and attributes in text based on a provided schema. + +## Entity Types + +These are the types of entities you should look for: + +{% for class_id, class_def in classes.items() %} +- **{{class_id}}**{% if class_def.subclass_of %} (subclass of {{class_def.subclass_of}}){% endif %}{% if class_def.comment %}: {{class_def.comment}}{% endif %} +{% endfor %} + +## Relationships + +These relationships connect entities to other entities: + +{% for prop_id, prop_def in object_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Attributes + +These attributes describe entity properties (text, numbers, etc.): + +{% for prop_id, prop_def in datatype_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Text to Analyze + +{{text}} + +## Your Task + +Extract the following from the text above: + +1. **Entities**: Things mentioned in the text and their types +2. **Relationships**: How entities relate to each other +3. **Attributes**: Properties of entities (like quantities, descriptions, etc.) + +## Output Format + +Return ONLY valid JSON (no markdown, no code blocks, no backticks) with this structure: + +``` +{ + "entities": [ + { + "entity": "entity name as it appears in text", + "type": "EntityType" + } + ], + "relationships": [ + { + "subject": "subject entity name", + "subject-type": "SubjectType", + "relation": "relationship_name", + "object": "object entity name", + "object-type": "ObjectType" + } + ], + "attributes": [ + { + "entity": "entity name", + "entity-type": "EntityType", + "attribute": "attribute_name", + "value": "literal value" + } + ] +} +``` + +## Important Rules + +1. **Entity names**: Use the exact text as it appears (e.g., "Cornish pasty", "beef") +2. **Types**: Use the EXACT type identifiers from the schema above (e.g., "fo/Recipe", "fo/Food") +3. **Relationships**: Use the EXACT relationship names from the schema (e.g., "fo/has_ingredient") +4. **Attributes**: Use the EXACT attribute names from the schema (e.g., "fo/serves") +5. **Type disambiguation**: If the same entity name appears with different types, list it separately for each type +6. **All arrays can be empty** if nothing is found + +## Example + +Input text: "Cornish pasty is a savory pastry filled with beef and potatoes. This recipe serves 4 people." + +Expected output: +{ + "entities": [ + { + "entity": "Cornish pasty", + "type": "fo/Recipe" + }, + { + "entity": "beef", + "type": "fo/Food" + }, + { + "entity": "potatoes", + "type": "fo/Food" + } + ], + "relationships": [ + { + "subject": "Cornish pasty", + "subject-type": "fo/Recipe", + "relation": "fo/has_ingredient", + "object": "beef", + "object-type": "fo/Food" + }, + { + "subject": "Cornish pasty", + "subject-type": "fo/Recipe", + "relation": "fo/has_ingredient", + "object": "potatoes", + "object-type": "fo/Food" + } + ], + "attributes": [ + { + "entity": "Cornish pasty", + "entity-type": "fo/Recipe", + "attribute": "fo/serves", + "value": "4 people" + } + ] +} + +Now extract entities, relationships, and attributes from the text above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/openai.jsonnet new file mode 100644 index 00000000..5d232337 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/openai.jsonnet @@ -0,0 +1,42 @@ +// For OpenAI LLMs. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/schema-selection.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/schema-selection.txt new file mode 100644 index 00000000..39b180e5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/schema-selection.txt @@ -0,0 +1,25 @@ +You are a database schema selection expert. Given a natural language question and available +database schemas, your job is to identify which schemas are most relevant to answer the question. + +## Available Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}): {{ field.description }} +{% endfor %} + +{% endfor %} + +## Question: +{{ question }} + +## Instructions: +1. Analyze the question to understand what data is being requested +2. Examine each schema to understand what data it contains +3. Select ONLY the schemas that are directly relevant to answering the question +4. Return your answer as a JSON array of schema names + +## Response Format: +Return ONLY a JSON array of schema names, nothing else. +Example: ["customers", "orders", "products"] diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/slm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/slm.jsonnet new file mode 100644 index 00000000..48eb96d0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/prompts/slm.jsonnet @@ -0,0 +1,44 @@ +// For SLM. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/pulsar/pulsar-manager.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/pulsar/pulsar-manager.jsonnet new file mode 100644 index 00000000..c1ca4515 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/pulsar/pulsar-manager.jsonnet @@ -0,0 +1,40 @@ +local images = import "values/images.jsonnet"; + +{ + + "pulsar" +: { + + create:: function(engine) + +// FIXME: Should persist something? +// local volume = engine.volume(...) + + local container = + engine.container("pulsar") + .with_image(images.pulsar_manager) + .with_environment({ + SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties", + }) + .with_limits("0.5", "1.4G") + .with_reservations("0.1", "1.4G") + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2"); + + local containerSet = engine.containers( + "pulsar", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2); + + engine.resources([ + containerSet, + service, + ]) + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/pulsar/pulsar.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/pulsar/pulsar.jsonnet new file mode 100644 index 00000000..d417322d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/pulsar/pulsar.jsonnet @@ -0,0 +1,222 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +// This is a Pulsar configuration. Non-standalone mode so we deploy +// individual components: bookkeeper, broker and zookeeper. +// +// This also deploys the TrustGraph 'admin' container which initialises +// TrustGraph-specific namespaces etc. + +{ + + "pulsar" +: { + + // Zookeeper memory settings (can be overridden by memory-profile) + "zk-memory-limit":: "512M", + "zk-memory-reservation":: "512M", + "zk-heap":: "256m", + "zk-direct-memory":: "256m", + + // Bookie memory settings (can be overridden by memory-profile) + "bookie-memory-limit":: "1024M", + "bookie-memory-reservation":: "1024M", + "bookie-heap":: "256m", + "bookie-direct-memory":: "256m", + + // Broker memory settings (can be overridden by memory-profile) + "broker-memory-limit":: "800M", + "broker-memory-reservation":: "800M", + "broker-heap":: "384m", + "broker-direct-memory":: "384m", + + // Pulsar-init memory settings (can be overridden by memory-profile) + "init-memory-limit":: "256M", + "init-memory-reservation":: "256M", + "init-heap":: "128m", + "init-direct-memory":: "128m", + + create:: function(engine) + + // Capture memory settings into locals (self refers to pulsar object here) + local zkMemLimit = self["zk-memory-limit"]; + local zkMemReserv = self["zk-memory-reservation"]; + local zkHeap = self["zk-heap"]; + local zkDirect = self["zk-direct-memory"]; + + local bookieMemLimit = self["bookie-memory-limit"]; + local bookieMemReserv = self["bookie-memory-reservation"]; + local bookieHeap = self["bookie-heap"]; + local bookieDirect = self["bookie-direct-memory"]; + + local brokerMemLimit = self["broker-memory-limit"]; + local brokerMemReserv = self["broker-memory-reservation"]; + local brokerHeap = self["broker-heap"]; + local brokerDirect = self["broker-direct-memory"]; + + local initMemLimit = self["init-memory-limit"]; + local initMemReserv = self["init-memory-reservation"]; + local initHeap = self["init-heap"]; + local initDirect = self["init-direct-memory"]; + + // Zookeeper volume + local zkVolume = engine.volume("zookeeper").with_size("1G"); + + // Zookeeper container + local zkContainer = + engine.container("zookeeper") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper" + ]) + .with_limits("1", zkMemLimit) + .with_reservations("0.05", zkMemReserv) + .with_user("0:1000") + .with_volume_mount(zkVolume, "/pulsar/data/zookeeper") + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + zkHeap, zkHeap, zkDirect, + ], + }) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Pulsar cluster init container + local initContainer = + engine.container("pulsar-init") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650", + ]) + .with_limits("1", initMemLimit) + .with_reservations("0.05", initMemReserv) + .with_environment({ + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + initHeap, initHeap, initDirect, + ], + }); + + + // Bookkeeper volume + local bookieVolume = engine.volume("bookie").with_size("20G"); + + // Bookkeeper container + local bookieContainer = + engine.container("bookie") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + // false ^ causes this to be a 'failure' exit. + ]) + .with_limits("1", bookieMemLimit) + .with_reservations("0.1", bookieMemReserv) + .with_user("0:1000") + .with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper") + .with_environment({ + "clusterName": "cluster-a", + "zkServers": "zookeeper:2181", + "bookieId": "bookie", + "metadataStoreUri": "metadata-store:zk:zookeeper:2181", + "advertisedAddress": "bookie", + "BOOKIE_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + bookieHeap, bookieHeap, bookieDirect, + ], + }) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker, stateless (uses ZK and Bookkeeper for state) + local brokerContainer = + engine.container("pulsar") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" + ]) + .with_limits("1", brokerMemLimit) + .with_reservations("0.1", brokerMemReserv) + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "zookeeperServers": "zookeeper:2181", + "clusterName": "cluster-a", + "managedLedgerDefaultEnsembleSize": "1", + "managedLedgerDefaultWriteQuorum": "1", + "managedLedgerDefaultAckQuorum": "1", + "advertisedAddress": "pulsar", + "advertisedListeners": "external:pulsar://pulsar:6650,localhost:pulsar://localhost:6650", + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + brokerHeap, brokerHeap, brokerDirect, + ], + }) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + // Container sets + local zkContainerSet = engine.containers( + "zookeeper", + [ + zkContainer, + ] + ); + + local initContainerSet = engine.containers( + "init-pulsar", + [ + initContainer, + ] + ); + + local bookieContainerSet = engine.containers( + "bookie", + [ + bookieContainer, + ] + ); + + local brokerContainerSet = engine.containers( + "pulsar", + [ + brokerContainer, + ] + ); + + // Zookeeper service + local zkService = + engine.service(zkContainerSet) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Bookkeeper service + local bookieService = + engine.service(bookieContainerSet) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker service + local brokerService = + engine.service(brokerContainerSet) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + engine.resources([ + zkVolume, + bookieVolume, + zkContainerSet, + initContainerSet, + bookieContainerSet, + brokerContainerSet, + zkService, + bookieService, + brokerService, + ]) + + } + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-additionals.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-additionals.jsonnet new file mode 100644 index 00000000..3e9f3481 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-additionals.jsonnet @@ -0,0 +1,134 @@ +local decode = import "decode-config.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Custom engine that collects configVolume parts +local engine = { + + // Collection of all configVolume parts + configVolumes:: [], + + // Implement all required engine methods as no-ops + container:: function(name) { + with_image:: function(x) self, + with_command:: function(x) self, + with_environment:: function(x) self, + with_limits:: function(c, m) self, + with_reservations:: function(c, m) self, + with_port:: function(src, dest, name) self, + with_volume_mount:: function(vol, mnt) self, + with_user:: function(x) self, + with_runtime:: function(x) self, + with_privileged:: function(x) self, + with_ipc:: function(x) self, + with_capability:: function(x) self, + with_device:: function(hdev, cdev) self, + with_env_var_secrets:: function(vars) self, + }, + + volume:: function(name) { + with_size:: function(size) self, + }, + + // The key method - collects configVolume parts + configVolume:: function(name, dir, parts) + local collector = self + { + configVolumes: super.configVolumes + [ + { + dir: dir, + parts: parts, + } + ] + }; + { + // Return a dummy volume that has the collector in it + name: name, + with_size:: function(size) collector, + // Provide a way to get back to the collector + getCollector:: function() collector, + }, + + secretVolume:: function(name, dir, parts) { + with_size:: function(size) self, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self, + }, + + containers:: function(name, containers) self, + + service:: function(containers) { + with_port:: function(src, dest, name) self, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self, + }, + + resources:: function(res) + // Fold over resources and collect any configVolume state + local collected = std.foldl( + function(state, r) + if std.objectHasAll(r, 'getCollector') then + // Merge the configVolumes from the volume's collector into our state + local volumeCollector = r.getCollector(); + state + { + configVolumes: state.configVolumes + volumeCollector.configVolumes + } + else + state, + res, + self + ); + collected, +}; + +// Execute all component create() functions with our collecting engine +// Note: create:: is a hidden field, so we must use objectHasAll not objectHas +local result = std.foldl( + function(state, p) + if std.objectHasAll(p, 'create') then + // Pattern has create directly - call it + p.create(state) + else + state, + std.objectValues(patterns), + engine +); + +// Debug: show what we collected +local debug = { + numPatterns: std.length(std.objectValues(patterns)), + numConfigVolumes: std.length(result.configVolumes), +}; + +// Transform collected data into output format +local allFiles = std.flattenArrays([ + [ + { + // Remove trailing slash from dir to avoid double slashes + path: std.join("/", [std.rstripChars(cv.dir, "/"), filename]), + content: cv.parts[filename] + } + for filename in std.objectFields(cv.parts) + ] + for cv in result.configVolumes +]); + +// Deduplicate by path - use a map to keep only unique paths +local uniqueMap = std.foldl( + function(acc, item) acc + { [item.path]: item }, + allFiles, + {} +); + +// Convert back to array +local additionals = std.objectValues(uniqueMap); + +// Output the array +additionals diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-aks-k8s.jsonnet new file mode 100644 index 00000000..bcec1cbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-aks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/aks-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-docker-compose.jsonnet new file mode 100644 index 00000000..7f0dbabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-docker-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/docker-compose.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-eks-k8s.jsonnet new file mode 100644 index 00000000..f9de59cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-eks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/eks-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-gcp-k8s.jsonnet new file mode 100644 index 00000000..50037a5c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-gcp-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/gcp-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-minikube-k8s.jsonnet new file mode 100644 index 00000000..6fa64706 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-minikube-k8s.jsonnet @@ -0,0 +1,26 @@ + +local engine = import "../engine/minikube-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +// Extract resources using the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-noop.jsonnet new file mode 100644 index 00000000..3ad735d9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-noop.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/noop.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-ovh-k8s.jsonnet new file mode 100644 index 00000000..92f61041 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-ovh-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/ovh-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-podman-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-podman-compose.jsonnet new file mode 100644 index 00000000..7f0dbabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-podman-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/docker-compose.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-scw-k8s.jsonnet new file mode 100644 index 00000000..4f6c1d7a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-scw-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/scw-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-tg-configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-tg-configuration.jsonnet new file mode 100644 index 00000000..59b4732a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/config-to-tg-configuration.jsonnet @@ -0,0 +1,15 @@ + +local engine = import "../engine/noop.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract configuration directly from patterns +patterns.configuration.configuration + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/decode-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/decode-config.jsonnet new file mode 100644 index 00000000..759513c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/renderers/decode-config.jsonnet @@ -0,0 +1,31 @@ + +local components = import "../components.jsonnet"; + +local apply = function(p, components) + + local base = { + + with:: function(k, v) self + { + [k] +:: v + }, + + with_params:: function(pars) + self + std.foldl( + function(obj, par) obj.with(par.key, par.value), + std.objectKeysValues(pars), + self + ), + + }; + + local component = base + components[p.name]; + + component.with_params(p.parameters); + +local decode = function(config) + local add = function(state, c) state + apply(c, components); + local patterns = std.foldl(add, config, {}); + patterns; + +decode + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/row-store/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/row-store/cassandra.jsonnet new file mode 100644 index 00000000..7fdf433e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/row-store/cassandra.jsonnet @@ -0,0 +1,77 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "backends/cassandra.jsonnet"; + +cassandra + { + + "store-objects" +: { + + create:: function(engine) + + local container = + engine.container("store-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "objects-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-objects" +: { + + create:: function(engine) + + local container = + engine.container("query-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "objects-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/config-composer.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/config-composer.jsonnet new file mode 100644 index 00000000..e93ff044 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/config-composer.jsonnet @@ -0,0 +1,97 @@ +// Configuration Composer Module +// Orchestrates the complete configuration building process +// Combines all components into the final TrustGraph configuration + +local flow_builder = import "flow-builder.jsonnet"; +local interface_builder = import "interface-builder.jsonnet"; + +{ + // Main function to build the complete configuration + build: function(config_spec) + // Extract configuration parameters + local flow_blueprints = config_spec.flow_blueprints; + local default_flow_blueprint = config_spec.default_flow_blueprint; + local default_flow_id = config_spec.default_flow_id; + local flow_init_parameters = config_spec.flow_init_parameters; + + // Build all processors for the default flow + local blueprint_processors = flow_builder.build_blueprint_processors( + flow_blueprints, + default_flow_blueprint, + flow_init_parameters + ); + + local flow_processors = flow_builder.build_flow_processors( + flow_blueprints, + default_flow_blueprint, + default_flow_id, + flow_init_parameters + ); + + // Combine processors into flow objects + local processor_array = blueprint_processors + flow_processors; + local flow_objects = flow_builder.build_flow_objects(processor_array); + local active_flows = flow_builder.merge_flow_objects(flow_objects); + + // Build interfaces for the default flow + local default_flow_interfaces = interface_builder.build_interfaces( + flow_blueprints, + default_flow_blueprint, + default_flow_id, + flow_init_parameters + ); + + // Return object with nested configuration (for backwards compatibility) + { + // Create function (for backwards compatibility) + create: function(engine) {}, + + // The actual configuration object + configuration: { + // Prompts configuration + prompt: { + "system": config_spec.prompts["system-template"], + "template-index": std.objectFieldsAll(config_spec.prompts.templates), + } + { + ["template." + template.key]: template.value + for template in std.objectKeysValuesAll(config_spec.prompts.templates) + }, + + // Tools configuration + tool: { + [tool.id]: tool + for tool in config_spec.tools + }, + + // MCP configuration + mcp: config_spec.mcp, + + // Flow blueprints reference + "flow-blueprint": flow_blueprints, + + // Interface descriptions + "interface-description": config_spec.interface_descriptions, + + // Flow instances + "flow": { + [default_flow_id]: { + "description": "Default processing flow", + "blueprint-name": default_flow_blueprint, + "interfaces": default_flow_interfaces, + "parameters": flow_init_parameters, + }, + }, + + // Active flow processors + "active-flow": active_flows, + + // Token costs and parameter types + "token-cost": config_spec.token_costs, + "parameter-type": config_spec.parameter_types, + + // Collections configuration + "collection": config_spec.collection, + + }, + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/flow-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/flow-builder.jsonnet new file mode 100644 index 00000000..0b142750 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/flow-builder.jsonnet @@ -0,0 +1,72 @@ +// Flow Builder Module +// Processes flow blueprints and builds complete flow configurations +// Handles {blueprint}, {id}, and parameter substitutions + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds blueprint-level processors with parameter substitution + // Processes the 'blueprint' section of flow blueprints + build_blueprint_processors: function(flow_blueprints, blueprint_name, parameters) + [ + [ + // Replace {blueprint} in the processor key + local key = std.strReplace(processor.key, "{blueprint}", blueprint_name); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // First replace {blueprint}, then substitute parameters + local blueprint_replaced = std.strReplace(field.value, "{blueprint}", blueprint_name); + param_processor.substitute_parameters(blueprint_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_blueprints[blueprint_name].blueprint) + ], + + // Builds flow-level processors with parameter substitution + // Processes the 'flow' section of flow blueprints + build_flow_processors: function(flow_blueprints, blueprint_name, flow_id, parameters) + [ + [ + // Replace both {blueprint} and {id} in the processor key + local key = std.strReplace( + std.strReplace(processor.key, "{blueprint}", blueprint_name), + "{id}", flow_id + ); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // Replace {blueprint} and {id}, then substitute parameters + local blueprint_replaced = std.strReplace(field.value, "{blueprint}", blueprint_name); + local id_replaced = std.strReplace(blueprint_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_blueprints[blueprint_name].flow) + ], + + // Combines blueprint and flow processors into flow objects + build_flow_objects: function(processor_array) + std.map( + function(item) { + [item[0][0]] +: { + [item[0][1]]: item[1] + } + }, + processor_array + ), + + // Merges all flow objects into a single flows_active configuration + merge_flow_objects: function(flow_objects) + std.foldr( + function(a, b) a + b, + flow_objects, + {} + ), +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/interface-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/interface-builder.jsonnet new file mode 100644 index 00000000..2143e600 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/interface-builder.jsonnet @@ -0,0 +1,30 @@ +// Interface Builder Module +// Processes flow class interfaces with parameter substitution +// Handles both string interfaces and nested object interfaces + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds interfaces for a specific flow class and instance + // Processes the 'interfaces' section of flow classes + build_interfaces: function(flow_classes, class_name, flow_id, parameters) + local interface_spec = flow_classes[class_name].interfaces; + { + [interface.key]: + if std.isString(interface.value) then + // Simple string interface - apply all substitutions + local class_replaced = std.strReplace(interface.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + else + // Complex object interface - process nested fields + { + [field.key]: + local class_replaced = std.strReplace(field.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(interface.value) + } + for interface in std.objectKeysValuesAll(interface_spec) + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/interface-descriptions.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/interface-descriptions.jsonnet new file mode 100644 index 00000000..f9dda8b0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/interface-descriptions.jsonnet @@ -0,0 +1,89 @@ +// Interface Descriptions Module +// Defines all external interfaces available in TrustGraph flows +// These are the 'endpoints' that external systems can interact with + +{ + // Document loading interfaces - for data ingestion + "document-load": { + "description": "Document loader", + "kind": "send", + "visible": true, + }, + "text-load": { + "description": "Text document loader", + "kind": "send", + "visible": true, + }, + + // Data storage interfaces - for processed data streams + "entity-contexts-load": { + "description": "Entity contexts loader", + "kind": "send", + }, + "triples-store": { + "description": "Triples loader", + "kind": "send", + }, + "graph-embeddings-store": { + "description": "Graph embeddings loader", + "kind": "send", + }, + "document-embeddings-store": { + "description": "Document embeddings loader", + "kind": "send", + }, + "objects-store": { + "description": "Object store", + "kind": "request-response", + }, + + // Query interfaces - for retrieving information + "graph-rag": { + "description": "GraphRAG service", + "kind": "request-response", + }, + "document-rag": { + "description": "ChunkRAG service", + "kind": "request-response", + }, + "triples": { + "description": "Triples query service", + "kind": "request-response", + }, + "graph-embeddings": { + "description": "Graph embeddings service", + "kind": "request-response", + }, + "document-embeddings": { + "description": "Document embeddings service", + "kind": "request-response", + }, + "objects": { + "description": "Object query service", + "kind": "request-response", + }, + + // Processing services - for text and data processing + "prompt": { + "description": "Prompt service", + "kind": "request-response", + }, + "agent": { + "description": "Agent service", + "kind": "request-response", + }, + "text-completion": { + "description": "Text completion service", + "kind": "request-response", + }, + + // Query translation services - for natural language queries + "nlp-query": { + "description": "NLP question to GraphQL service", + "kind": "request-response", + }, + "structured-query": { + "description": "Structured query service", + "kind": "request-response", + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/parameter-processor.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/parameter-processor.jsonnet new file mode 100644 index 00000000..f317ae9f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/parameter-processor.jsonnet @@ -0,0 +1,38 @@ +// Parameter Processing Module +// Handles dynamic parameter replacement in configuration values +// Replaces {parameter_name} placeholders with actual parameter values + +{ + // Applies parameter substitutions to string values + // Only processes strings - leaves other types unchanged + substitute_parameters: function(value, parameters) + if std.isString(value) then + std.foldl( + function(acc, param) + // Only do string replacement if param.value is a string + if std.isString(param.value) then + std.strReplace(acc, "{" + param.key + "}", param.value) + else + acc, // Skip replacement for non-string parameter values + std.objectKeysValuesAll(parameters), + value + ) + else + value, + + // Applies parameter substitutions to all values in an object + // Recursively processes nested objects and arrays + substitute_parameters_in_object: function(obj, parameters) + if std.isObject(obj) then + { + [key]: $.substitute_parameters_in_object(obj[key], parameters) + for key in std.objectFields(obj) + } + else if std.isArray(obj) then + [ + $.substitute_parameters_in_object(item, parameters) + for item in obj + ] + else + $.substitute_parameters(obj, parameters), +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/tools.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/tools.jsonnet new file mode 100644 index 00000000..a84cd0bb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/tools.jsonnet @@ -0,0 +1,52 @@ +// Tools Configuration Module +// Defines all available tools that can be used by agents and flows +// Each tool specifies its interface, arguments, and behavior + +[ + // Knowledge extraction tool - extracts structured knowledge from text + { + id: "knowledge-extraction", + name: "Knowledge extraction", + description: "Takes a chunk of text and extracts knowledge in definition and relationship formats. The input is a text chunk", + type: "prompt", + template: "agent-kg-extract", + arguments: [ + { + "name": "text", + "type": "string", + "description": "The text chunk", + } + ], + }, + + // Knowledge query tool - queries the knowledge base + { + id: "knowledge-query", + name: "Knowledge query", + description: "This tool queries a knowledge base that holds information about domain-specific information. The question should be a natural language question.", + type: "knowledge-query", + collection: "default", + arguments: [ + { + name: "question", + type: "string", + description: "A simple natural language question.", + } + ] + }, + + // LLM completion tool - general purpose text completion + { + id: "llm-completion", + name: "LLM text completion", + type: "text-completion", + description: "This tool queries an LLM for non-domain-specific information. The question should be a natural language question.", + arguments: [ + { + name: "question", + type: "string", + description: "The question which should be asked of the LLM.", + } + ] + } +] \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/trustgraph-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/trustgraph-config.jsonnet new file mode 100644 index 00000000..d860ab3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/runtime-config/trustgraph-config.jsonnet @@ -0,0 +1,90 @@ +// TrustGraph Main Configuration +// Clean, modular composition of TrustGraph configuration +// Uses specialized modules for different aspects of config building + +// Import dependencies +local images = import "../values/images.jsonnet"; +local url = import "../values/url.jsonnet"; +local prompts = import "../prompts/mixtral.jsonnet"; +local default_prompts = import "../prompts/default-prompts.jsonnet"; +local token_costs = import "../values/token-costs.jsonnet"; +local flow_blueprints = import "../flows/flow-blueprints.jsonnet"; +local config_composer = import "config-composer.jsonnet"; +local interface_descriptions = import "interface-descriptions.jsonnet"; +local tools = import "tools.jsonnet"; +local temperature_params = import "../parameters/temperature-param-types.jsonnet"; +local chunking_params = import "../parameters/chunking-param-types.jsonnet"; + +// Main configuration object +local configuration = { + + // Prompt templates + prompts:: default_prompts, + + // Tool definitions + tools:: tools, + + // MCP configuration + mcp:: {}, + + // Flow classes reference + "flow-blueprints":: flow_blueprints, + + // LLM model parameters + "llm-models" +:: {}, + + // Embeddings model parameters + "embeddings-models" +:: {}, + + collections +:: { + "trustgraph:default": { + "user": "default-user", + "collection": "default", + "name": "Default Collection", + "description": "Default collection", + "tags": ["default"], + }, + }, + + // Default model and flow parameters + flow_init_parameters:: { + "llm-model": $["llm-models"].default, + "llm-rag-model": $["llm-models"].default, + "llm-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "llm-rag-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "chunk-size": std.toString($["parameter-types"]["chunk-size"].default), + "chunk-overlap": std.toString($["parameter-types"]["chunk-overlap"].default), + "embeddings-model": $["embeddings-models"].default, + }, + + // Interface descriptions for external endpoints + "interface-descriptions":: interface_descriptions, + + // Parameter type definitions + "parameter-types":: { + "llm-model": $["llm-models"], + "embeddings-model": $["embeddings-models"], + } + chunking_params + temperature_params, + + // Token costs + "token-costs":: token_costs, + + // Build the complete configuration using the composer + configuration:: config_composer.build({ + flow_blueprints: $["flow-blueprints"], + default_flow_blueprint: "everything", + default_flow_id: "default", + flow_init_parameters: $["flow_init_parameters"], + prompts: $["prompts"], + tools: $["tools"], + mcp: $["mcp"], + interface_descriptions: $["interface-descriptions"], + parameter_types: $["parameter-types"], + token_costs: $["token-costs"], + collection: $["collections"], + }), + +} + default_prompts; + +// Export the final configuration +configuration diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/cassandra.jsonnet new file mode 100644 index 00000000..05f4fd27 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/cassandra.jsonnet @@ -0,0 +1,77 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "backends/cassandra.jsonnet"; + +cassandra + { + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/falkordb.jsonnet new file mode 100644 index 00000000..b4bace2d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/falkordb.jsonnet @@ -0,0 +1,79 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local falkordb = import "backends/falkordb.jsonnet"; + +falkordb + { + + "falkordb-url":: "falkor://falkordb:6379", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/memgraph.jsonnet new file mode 100644 index 00000000..72597d72 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/memgraph.jsonnet @@ -0,0 +1,84 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local memgraph = import "backends/memgraph.jsonnet"; + +memgraph + { + + "memgraph-url":: "bolt://memgraph:7687", + "memgraph-database":: "memgraph", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/neo4j.jsonnet new file mode 100644 index 00000000..08904715 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/triple-store/neo4j.jsonnet @@ -0,0 +1,79 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local neo4j = import "backends/neo4j.jsonnet"; + +neo4j + { + + "neo4j-url":: "bolt://neo4j:7687", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/ui/workbench-ui.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/ui/workbench-ui.jsonnet new file mode 100644 index 00000000..f2048e47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/ui/workbench-ui.jsonnet @@ -0,0 +1,32 @@ +local images = import "values/images.jsonnet"; + +{ + + "workbench-ui" +: { + + create:: function(engine) + + local container = + engine.container("workbench-ui") + .with_image(images["workbench-ui"]) + .with_limits("0.1", "256M") + .with_reservations("0.1", "256M") + .with_port(8888, 8888, "ui"); + + local containerSet = engine.containers( + "workbench-ui", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8888, 8888, "ui"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/values/images.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/values/images.jsonnet new file mode 100644 index 00000000..8b8b75a0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/values/images.jsonnet @@ -0,0 +1,36 @@ +local version = import "version.jsonnet"; +{ + cassandra: "docker.io/cassandra:4.1.10", +// Not working +// ceph: "quay.io/ceph/daemon:latest-reef", + neo4j: "docker.io/neo4j:2025.08.0-community-bullseye", + pulsar: "docker.io/apachepulsar/pulsar:4.1.0", + pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.4.0", + etcd: "quay.io/coreos/etcd:v3.6.4", + minio: "docker.io/minio/minio:RELEASE.2025-09-07T16-13-09Z", + garage: "docker.io/dxflrs/garage:v2.1.0", + milvus: "docker.io/milvusdb/milvus:v2.5.17", + prometheus: "docker.io/prom/prometheus:v3.8.0", + grafana: "docker.io/grafana/grafana:12.3.0", + loki: "docker.io/grafana/loki:3.6.2", + trustgraph_base: "docker.io/trustgraph/trustgraph-base:" + version, + trustgraph_flow: "docker.io/trustgraph/trustgraph-flow:" + version, + trustgraph_ocr: "docker.io/trustgraph/trustgraph-ocr:" + version, + trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version, + trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version, + trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version, + trustgraph_mcp: "docker.io/trustgraph/trustgraph-mcp:" + version, + qdrant: "docker.io/qdrant/qdrant:v1.15.4", + memgraph_mage: "docker.io/memgraph/memgraph-mage:3.5", + memgraph_lab: "docker.io/memgraph/lab:3.5.0", + falkordb: "docker.io/falkordb/falkordb:v4.12.5", + "workbench-ui": "docker.io/trustgraph/workbench-ui:1.5.5", + "ddg-mcp-server": "docker.io/trustgraph/ddg-mcp-server:0.1.0", + "tgi-service-intel-xpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-xpu", + "tgi-service-cpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-cpu", + "tgi-service-gaudi": "ghcr.io/huggingface/text-generation-inference:sha-f140440-gaudi", + "vllm-service-intel-xpu": "docker.io/intel/vllm:0.8.0-xpu", + "vllm-service-gaudi": "docker.io/trustgraph/vllm-hpu:027f5645", + "vllm-service-nvidia": "docker.io/vllm/vllm-openai:latest", + "vllm-service-intel-battlemage": "docker.io/intelanalytics/ipex-llm-serving-xpu:0.2.0-b6", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/values/token-costs.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/values/token-costs.jsonnet new file mode 100644 index 00000000..84c70373 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/values/token-costs.jsonnet @@ -0,0 +1,102 @@ +{ + "mistral.mistral-large-2407-v1:0": { + "model_name": "mistral.mistral-large-2407-v1:0", + "input_price": 0.000004, + "output_price": 0.000012 + }, + "meta.llama3-1-405b-instruct-v1:0": { + "model_name": "meta.llama3-1-405b-instruct-v1:0", + "input_price": 0.00000532, + "output_price": 0.000016 + }, + "mistral.mixtral-8x7b-instruct-v0:1": { + "model_name": "mistral.mixtral-8x7b-instruct-v0:1", + "input_price": 0.00000045, + "output_price": 0.0000007 + }, + "meta.llama3-1-70b-instruct-v1:0": { + "model_name": "meta.llama3-1-70b-instruct-v1:0", + "input_price": 0.00000099, + "output_price": 0.00000099 + }, + "meta.llama3-1-8b-instruct-v1:0": { + "model_name": "meta.llama3-1-8b-instruct-v1:0", + "input_price": 0.00000022, + "output_price": 0.00000022 + }, + "anthropic.claude-3-haiku-20240307-v1:0": { + "model_name": "anthropic.claude-3-haiku-20240307-v1:0", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "anthropic.claude-3-5-sonnet-20240620-v1:0": { + "model_name": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "cohere.command-r-plus-v1:0": { + "model_name": "cohere.command-r-plus-v1:0", + "input_price": 0.0000030, + "output_price": 0.0000150 + }, + "ollama": { + "model_name": "ollama", + "input_price": 0, + "output_price": 0 + }, + "claude-3-haiku-20240307": { + "model_name": "claude-3-haiku-20240307", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "claude-3-5-sonnet-20240620": { + "model_name": "claude-3-5-sonnet-20240620", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "claude-3-opus-20240229": { + "model_name": "claude-3-opus-20240229", + "input_price": 0.000015, + "output_price": 0.000075 + }, + "claude-3-sonnet-20240229": { + "model_name": "claude-3-sonnet-20240229", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "command-r-08-202": { + "model_name": "command-r-08-202", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "c4ai-aya-23-8b": { + "model_name": "c4ai-aya-23-8b", + "input_price": 0, + "output_price": 0 + }, + "llama.cpp": { + "model_name": "llama.cpp", + "input_price": 0, + "output_price": 0 + }, + "gpt-4o": { + "model_name": "gpt-4o", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-2024-08-06": { + "model_name": "gpt-4o-2024-08-06", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "gpt-4o-2024-05-13": { + "model_name": "gpt-4o-2024-05-13", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-mini": { + "model_name": "gpt-4o-mini", + "input_price": 0.00000015, + "output_price": 0.0000006 + } +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/values/url.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/values/url.jsonnet new file mode 100644 index 00000000..c3d4ad97 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/values/url.jsonnet @@ -0,0 +1,7 @@ +{ + pulsar: "pulsar://pulsar:6650", + pulsar_admin: "http://pulsar:8080", + milvus: "http://milvus:19530", + qdrant: "http://qdrant:6333", + object_store: "garage:3900", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/vector-store/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/vector-store/milvus.jsonnet new file mode 100644 index 00000000..213027d1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/vector-store/milvus.jsonnet @@ -0,0 +1,146 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local milvus = import "backends/milvus.jsonnet"; + +milvus + { + + "store-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/vector-store/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/vector-store/pinecone.jsonnet new file mode 100644 index 00000000..3803bdcb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/vector-store/pinecone.jsonnet @@ -0,0 +1,160 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; + +{ + + "pinecone-cloud":: "aws", + "pinecone-region":: "us-east-1", + + "store-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/vector-store/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/vector-store/qdrant.jsonnet new file mode 100644 index 00000000..24d6b040 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/vector-store/qdrant.jsonnet @@ -0,0 +1,170 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local qdrant = import "backends/qdrant.jsonnet"; + +qdrant + { + + "store-graph-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/zip-readme.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/zip-readme.md new file mode 100644 index 00000000..0b117792 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.8/zip-readme.md @@ -0,0 +1,28 @@ + +Note! this is a subset of possible configurations, to generate your own +launch config use the config util... + +- Production: https://config-ui.demo.trustgraph.ai +- Early release: https://dev.config-ui.demo.trustgraph.ai + +The config util auto-generates deployment instructions for your +configuration, so that's the recommended way to deploy. + +---------------------------------------------------------------------------- + +These are launch configurations for TrustGraph. See https://trustgraph.ai for +the quickstart using docker compose. + +Hint for Linux: There are files here which get mounted as volumes inside +Docker Compose containers. This may trigger SELinux rules on your system, to +permit access insider the containers, use a command like this... + +chcon -Rt svirt_sandbox_file_t grafana/ prometheus/ + +The file vertexai/private.json is a placeholder for real GCP credentials if +you are using the VertexAI LLM. If you're using that in Docker Compose, +replace with your real credentials, and don't forget to permit access if you +are using Linux: + +chcon -Rt svirt_sandbox_file_t vertexai/ + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/README.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/README.md new file mode 100644 index 00000000..23039e9a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/README.md @@ -0,0 +1,125 @@ + +# TrustGraph template generation + +There are two utilities here: + +- `generate`: Generates a single Docker Compose launch configuration + based on configuration you provide. +- `generate-all`: Generates the release bundle for releases. You won't + need to use this unless you are managing releases. + +## `generate-all` + +Previously, this generates a full set of all vector DB / triple store / LLM +combinations, and put them in a single ZIP file. But this got out of +hand, so at the time of writing, this generates a single configuraton +using Qdrant vector DB, Ollama LLM support and Cassandra for a triple store. + +The combinations are contained withing the code, it takes two arguments: +- output ZIP file (is over-written) +- TrustGraph version number + +``` +templates/generate-all output.zip 0.18.11 +``` + +## `generate` + +This utility takes a configuration file describing the components to bundle, +and outputs a Docker Compose YAML file. + +### Input configuration + +The input configuration is a JSON file, an array of components to pull into +the configuration. For each component, there is a name and a (possibly empty) +object describing addtional parameters for that component. + +Example: + +``` +[ + { + "name": "cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "qdrant", + "parameters": {} + }, + { + "name": "embeddings-hf", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph", + "parameters": {} + }, + { + "name": "googleaistudio", + "parameters": { + "googleaistudio-temperature": 0.3, + "googleaistudio-max-output-tokens": 2048, + "googleaistudio-model": "gemini-1.5-pro-002" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 1000, + "chunk-overlap": 50 + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "agent-manager-react", + "parameters": {} + } +] +``` + +If you want to make your own configuration you could try changing the +configuration above: +- Components which are essential: pulsar, trustgraph, graph-rag, grafana, + agent-manager-react +- You need a triple store, one of: cassandra, memgraph, falkordb, neo4j +- You need a vector store, one of: qdrant, pinecone +- You need an LLM, one of: azure, azure-openai, bedrock, claude, cohere, + llamafile, ollama, openai, vertexai. +- You need an embeddings implementation, one of: embeddings-hf, + embeddings-ollama +- Optionally add the Workbench tool: workbench-ui + +Components have over-ridable parameters, look in the component definition +in `templates/components/` to see what you can override. + +### Invocation + +Two parameters: +- The output ZIP file +- The version number + +The configuration file described above is provided on standard input + +``` +templates/generate out.zip 0.18.9 < config.json +``` + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/cassandra.jsonnet new file mode 100644 index 00000000..60f262d3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/cassandra.jsonnet @@ -0,0 +1,50 @@ +local images = import "values/images.jsonnet"; + +{ + + "cassandra" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1000M", + "memory-reservation":: "1000M", + "heap":: "300M", + + create:: function(engine) + + // Capture memory settings into locals + local memLimit = self["memory-limit"]; + local memReserv = self["memory-reservation"]; + local heap = self["heap"]; + + local vol = engine.volume("cassandra").with_size("20G"); + + local container = + engine.container("cassandra") + .with_image(images.cassandra) + .with_environment({ + JVM_OPTS: "-Xms%s -Xmx%s -Dcassandra.skip_wait_for_gossip_to_settle=0" % [ + heap, heap, + ], + }) + .with_limits("1.0", memLimit) + .with_reservations("0.5", memReserv) + .with_port(9042, 9042, "cassandra") + .with_volume_mount(vol, "/var/lib/cassandra"); + + local containerSet = engine.containers( + "cassandra", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9042, 9042, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/ceph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/ceph.jsonnet new file mode 100644 index 00000000..42406a95 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/ceph.jsonnet @@ -0,0 +1,446 @@ + +// +// This majorly does not work +// + +// This configuration fails primarily because it tries to treat Ceph +// like a stateless web app. You are currently pointing mon host to a +// generic service name (ceph-mon), but you aren't telling the Monitor +// process to assume that service identity. + +// To make this work with the "Service Name" approach across any +// engine, you need to fix the binding logic and the messenger protocol. + +// 1. The ceph.conf Fix + +// Need to enable Messenger v2 (modern) and tell the cluster to use +// the service names for its initial quorum. + +//Change this section: +// Ini, TOML + +// [global] +// fsid = %s +// mon initial members = mon0 +// mon host = ceph-mon:6789 +// ... + +// To this: +// Ini, TOML + +// [global] +// fsid = %s +// # Use the actual service names as members +// mon initial members = ceph-mon +// # Explicitly use the Service name (VIP) +// mon host = ceph-mon +// # Force modern protocol +// ms_bind_msgr2 = true +// ms_bind_msgr1 = true + +// 2. The MON Environment & Command Fix + +// This is the most critical part. Your current config sets MON_IP: +// "0.0.0.0". This causes the MON to bind to the Pod IP, which breaks +// when the Pod restarts. You must force it to bind to the Service IP. + +// Update mon_env: +// Code snippet + +// local mon_env = cluster_env + { +// CEPH_DAEMON: "MON", +// MON_NAME: "mon0", +// # Remove MON_IP: "0.0.0.0" +// # Add these: +// MON_ADDR: "ceph-mon", // This says to resolve the service name +// }; + +// Update mon_container command: You are currently wiping the MON data +// on every start (rm -rf /var/lib/ceph/mon/*). Stop doing that. If you +// wipe the data, you lose the cluster state and the OSDs will refuse to +// talk to the "new" MON. + +// Code snippet + +// .with_command([ +// "bash", "-c", +// # 1. Resolve the Service IP at runtime +// "export MON_IP=$(getent hosts ceph-mon | awk '{ print $1 }'); " + +// inject_mon_config + +// # 2. Start the daemon telling it its PUBLIC address is the Service VIP +// "exec /opt/ceph-container/bin/entrypoint.sh" +// ]) + +// 3. Why your current config "Majorly does not work" + +// The "Wipe" Logic: By running rm -rf /var/lib/ceph/mon/* in the +// MON container, you are creating a "New Cluster" every time the +// container starts. Since the OSDs store the fsid and cluster +// secrets, they will reject the "new" MON. + +// DNS Race Condition: Your OSD/MGR/RGW containers wait for +// ceph-mon DNS, which is good. However, if ceph-mon resolves to a +// Round Robin IP (multiple pods) rather than a stable ClusterIP, the +// connection will be flaky. + +// Messenger Protocol: Without ms_bind_msgr2, Ceph defaults to the +// old v1 protocol which is much more sensitive to NAT/Container IP +// mismatches. + +// SUMMARY + +// Component: ceph.conf +// Change: Add ms_bind_msgr2 = true +// Why: Supports modern container networking better. + +// Component: MON Start +// Change: Remove rm -rf +// Why: Ceph MONs must keep their database to maintain the cluster. + +// Component: MON Address +// Change: Use getent hosts ceph-mon +// Why: Forces the MON to advertise the Service VIP instead of its own Pod IP. + +// Component: MON Keyring +// Change: Ensure /etc/ceph/ceph.mon.keyring exists +// Why: MONs need their specific key to start. + +local images = import "values/images.jsonnet"; + +{ + with:: function(key, value) + self + { + ["ceph-" + key]:: value, + }, + + // Ceph credentials and cluster settings + "ceph-access-key":: "object-user", + "ceph-secret-key":: "object-password", + "ceph-cluster-id":: "ceph", + "ceph-fsid":: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + + // Pool redundancy settings + // size: 2 = two replicas for fault tolerance + // min_size: 1 = allow degraded I/O if one OSD is down (prevents cluster freeze) + "ceph-pool-size":: "2", + "ceph-pool-min-size":: "1", + + ceph +: { + create:: function(engine) + // Pre-Shared Cryptographic Material - Config-as-Code Approach + // These keys are generated once and distributed to all daemons + // This ensures cryptographic consistency across the shared-nothing architecture + local admin_key = "AQBpxSBlAAAAABAAU99V6D8vS7Uu9y1S8W0iBg=="; + local mon_key = "AQBpxSBlAAAAABAAn7pL/pG9oT+X6vO7V1S6bg=="; + + // Ceph configuration file - rendered from Jsonnet variables + local ceph_conf = ||| + [global] + fsid = %s + mon initial members = mon0 + mon host = ceph-mon:6789 + public network = 0.0.0.0/0 + cluster network = 0.0.0.0/0 + osd pool default size = %s + osd pool default min size = %s + osd crush chooseleaf type = 0 + auth cluster required = cephx + auth service required = cephx + auth client required = cephx + ||| % [$["ceph-fsid"], $["ceph-pool-size"], $["ceph-pool-min-size"]]; + + // Admin keyring - distributed to all daemons + local admin_keyring = ||| + [client.admin] + key = %s + caps mds = "allow *" + caps mgr = "allow *" + caps mon = "allow *" + caps osd = "allow *" + ||| % [admin_key]; + + // Monitor keyring - used by MON for cluster operations + local mon_keyring = ||| + [mon.] + key = %s + caps mon = "allow *" + ||| % [mon_key]; + + // Config injection command - writes files before entrypoint + local inject_config = "printf '%s' > /etc/ceph/ceph.conf; printf '%s' > /etc/ceph/ceph.client.admin.keyring; " % [ceph_conf, admin_keyring]; + local inject_mon_config = inject_config + ("printf '%s' > /etc/ceph/ceph.mon.keyring; " % [mon_keyring]); + + // Data volumes - sized appropriately for production workloads + local vol_mon = engine.volume("ceph-mon").with_size("20G"); + local vol_mgr = engine.volume("ceph-mgr").with_size("20G"); + local vol_osd = engine.volume("ceph-osd").with_size("100G"); + local vol_rgw = engine.volume("ceph-rgw").with_size("20G"); + + // Isolated config volumes per daemon (ReadWriteOnce compatible) + // Each daemon gets its own non-shared config volume to support + // multi-node scheduling in K8s and other orchestrators + local vol_mon_config = engine.volume("ceph-mon-config").with_size("500M"); + local vol_mgr_config = engine.volume("ceph-mgr-config").with_size("500M"); + local vol_osd_config = engine.volume("ceph-osd-config").with_size("500M"); + local vol_rgw_config = engine.volume("ceph-rgw-config").with_size("500M"); + local vol_init_config = engine.volume("ceph-init-config").with_size("500M"); + + // Simplified cluster environment - Config-as-Code model + // No fetch logic needed - config is injected before entrypoint runs + local cluster_env = { + CLUSTER: $["ceph-cluster-id"], + FSID: $["ceph-fsid"], + KV_TYPE: "none", // No external coordination + }; + + // MON-specific environment + // Config-as-Code: MON uses injected config files, not fetch logic + // + // CRITICAL: MON_DATA_AVAIL="0" forces fresh cluster bootstrap + // The ceph/daemon entrypoint script (variables_stack.sh) uses this as a gate: + // - MON_DATA_AVAIL="0" -> run mkfs, create new cluster with our FSID + // - MON_DATA_AVAIL="1" -> attempt to join existing cluster (infinite probe loop) + // + // Network configuration for monmap generation + local mon_env = cluster_env + { + CEPH_DAEMON: "MON", + MON_NAME: "mon0", + MON_PORT: "6789", + MON_DATA_AVAIL: "0", + MON_IP: "0.0.0.0", + NETWORK_AUTO_DETECT: "4", + CEPH_PUBLIC_NETWORK: "0.0.0.0/0", + }; + + // Simplified daemon environments - Config-as-Code model + // All daemons receive config via injection, not fetch from MON + // This eliminates "static mode" errors and networking complexity + + // MGR-specific environment + local mgr_env = cluster_env + { + CEPH_DAEMON: "MGR", + MGR_NAME: "mgr0", + }; + + // OSD-specific environment + local osd_env = cluster_env + { + CEPH_DAEMON: "OSD", + OSD_TYPE: "directory", + }; + + // RGW-specific environment + local rgw_env = cluster_env + { + CEPH_DAEMON: "RGW", + RGW_NAME: "rgw0", + RGW_FRONTEND_PORT: "7480", + }; + + // MON (Monitor) container - cluster state and quorum + // Config-as-Code: Injects pre-shared keys before entrypoint + // CRITICAL: Wipes /var/lib/ceph/mon/* on every start to force fresh bootstrap + // This ensures MON always uses our FSID and doesn't inherit stale cluster state + local mon_container = + engine.container("ceph-mon") + .with_image(images.ceph) + .with_environment(mon_env) + .with_command([ + "bash", "-c", + "rm -rf /var/lib/ceph/mon/*; " + + inject_mon_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(6789, 6789, "mon") + .with_port(3300, 3300, "mon-msgr2") + .with_volume_mount(vol_mon, "/var/lib/ceph/mon") + .with_volume_mount(vol_mon_config, "/etc/ceph"); + + // MGR (Manager) container - cluster management and dashboard + // Config-as-Code: Uses injected config files with pre-shared keys + // DNS wait ensures MON is available before MGR connects + local mgr_container = + engine.container("ceph-mgr") + .with_image(images.ceph) + .with_environment(mgr_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(7000, 7000, "mgr") + .with_port(8443, 8443, "dashboard") + .with_port(9283, 9283, "prometheus") + .with_volume_mount(vol_mgr, "/var/lib/ceph/mgr") + .with_volume_mount(vol_mgr_config, "/etc/ceph"); + + // OSD (Object Storage Daemon) - actual data storage + // Config-as-Code: Uses injected config files with pre-shared keys + // Increased resources to prevent OOM during recovery operations + // DNS wait ensures MON is available before OSD connects + local osd_container = + engine.container("ceph-osd") + .with_image(images.ceph) + .with_environment(osd_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("2.0", "4096M") + .with_reservations("1.0", "2048M") + .with_port(6800, 6800, "osd") + .with_volume_mount(vol_osd, "/var/lib/ceph/osd") + .with_volume_mount(vol_osd_config, "/etc/ceph"); + + // RGW (RADOS Gateway) - S3 API endpoint + // Config-as-Code: Uses injected config files with pre-shared keys + // DNS wait ensures MON is available before RGW connects + local rgw_container = + engine.container("ceph-rgw") + .with_image(images.ceph) + .with_environment(rgw_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(7480, 7480, "s3") + .with_volume_mount(vol_rgw, "/var/lib/ceph/radosgw") + .with_volume_mount(vol_rgw_config, "/etc/ceph"); + + // Init container - one-time S3 user provisioning + // IMPORTANT: This container exits with code 0 after completion + // Orchestrator must NOT restart it (use K8s Job or Compose restart: "no") + // Config-as-Code: Uses injected config to run radosgw-admin commands + local init_container = + engine.container("ceph-init") + .with_image(images.ceph) + .with_environment({ + CLUSTER: $["ceph-cluster-id"], + FSID: $["ceph-fsid"], + KV_TYPE: "none", + RGW_ACCESS_KEY: $["ceph-access-key"], + RGW_SECRET_KEY: $["ceph-secret-key"], + }) + .with_limits("0.5", "512M") + .with_reservations("0.25", "256M") + .with_volume_mount(vol_init_config, "/etc/ceph") + .with_command([ + "bash", "-c", + inject_config + ||| + set -e + + # Wait for cluster health + echo "Waiting for Ceph cluster to be healthy..." + MAX_ATTEMPTS=60 + ATTEMPT=0 + until ceph --cluster ${CLUSTER} health 2>/dev/null | grep -q "HEALTH_OK\|HEALTH_WARN"; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: Cluster failed to become healthy after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: Cluster not ready, retrying in 5s..." + sleep 5 + done + echo "Cluster is healthy." + + # Wait for RGW availability + echo "Waiting for RGW to be ready..." + ATTEMPT=0 + until curl -sf http://ceph-rgw:7480 >/dev/null 2>&1; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: RGW failed to become ready after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: RGW not ready, retrying in 5s..." + sleep 5 + done + echo "RGW is ready." + + # Idempotent S3 user creation + echo "Provisioning S3 user: ${RGW_ACCESS_KEY}" + if radosgw-admin --cluster ${CLUSTER} user info --uid="${RGW_ACCESS_KEY}" >/dev/null 2>&1; then + echo "User ${RGW_ACCESS_KEY} already exists, skipping creation." + else + echo "Creating new S3 user: ${RGW_ACCESS_KEY}" + radosgw-admin --cluster ${CLUSTER} user create \ + --uid="${RGW_ACCESS_KEY}" \ + --display-name="Object Storage User" \ + --access-key="${RGW_ACCESS_KEY}" \ + --secret-key="${RGW_SECRET_KEY}" + echo "S3 user created successfully." + fi + + echo "Initialization complete. Exiting." + exit 0 + |||, + ]); + + // Container sets - each daemon gets its own for K8s node distribution + local mon_containerSet = engine.containers("ceph-mon", [mon_container]); + local mgr_containerSet = engine.containers("ceph-mgr", [mgr_container]); + local osd_containerSet = engine.containers("ceph-osd", [osd_container]); + local rgw_containerSet = engine.containers("ceph-rgw", [rgw_container]); + local init_containerSet = engine.containers("ceph-init", [init_container]); + + // Services - expose daemon ports for inter-daemon communication + local mon_service = + engine.service(mon_containerSet) + .with_port(6789, 6789, "mon") + .with_port(3300, 3300, "mon-msgr2"); + + local mgr_service = + engine.service(mgr_containerSet) + .with_port(7000, 7000, "mgr") + .with_port(8443, 8443, "dashboard") + .with_port(9283, 9283, "prometheus"); + + local osd_service = + engine.service(osd_containerSet) + .with_port(6800, 6800, "osd"); + + local rgw_service = + engine.service(rgw_containerSet) + .with_port(7480, 7480, "s3"); + + engine.resources([ + + // Data volumes + vol_mon, + vol_mgr, + vol_osd, + vol_rgw, + + // Config volumes (isolated, no sharing) + vol_mon_config, + vol_mgr_config, + vol_osd_config, + vol_rgw_config, + vol_init_config, + + // Container sets + mon_containerSet, + mgr_containerSet, + osd_containerSet, + rgw_containerSet, + init_containerSet, + + // Services + mon_service, + mgr_service, + osd_service, + rgw_service, + + ]) + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/falkordb.jsonnet new file mode 100644 index 00000000..1d4176d8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/falkordb.jsonnet @@ -0,0 +1,38 @@ +local images = import "values/images.jsonnet"; + +{ + + "falkordb" +: { + + create:: function(engine) + + local vol = engine.volume("falkordb").with_size("20G"); + + local container = + engine.container("falkordb") + .with_image(images.falkordb) + .with_limits("1.0", "768M") + .with_reservations("0.5", "768M") + .with_port(6379, 6379, "api") + .with_port(3010, 3000, "ui") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "falkordb", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6379, 6379, "api") + .with_port(3010, 3010, "ui"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/garage.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/garage.jsonnet new file mode 100644 index 00000000..9d339bfb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/garage.jsonnet @@ -0,0 +1,249 @@ +local images = import "values/images.jsonnet"; + +{ + + garage +: { + + // Garage S3 credentials - these are the actual access key ID and secret key + // Access Key ID must be in format: GK + 24 hex characters (12 bytes) + // Secret Key must be 64 hex characters (32 bytes) + // For production, generate secure random values and override these defaults + "access-key":: "GK000000000000000000000001", + "secret-key":: "b171f00be9be4c32c734f4c05fe64c527a8ab5eb823b376cfa8c2531f70fc427", + "rpc-secret":: "bbba746a9e289bad64a9e7a36a4299dac8d6e0b8cc2a6c2937fe756df4492008", + // For a production system, override this value + "admin-token":: "batts-rockhearted-unpartially", + region:: "garage", + "replication-factor":: "1", // Set to 1 for single-node, 3 for production + + // Storage volume sizes + "meta-size":: "2G", // Metadata volume size + "data-size":: "5G", // Data volume size (also used for cluster layout capacity) + + create:: function(engine) + + local accessKey = self["access-key"]; + local secretKey = self["secret-key"]; + local rpcSecret = self["rpc-secret"]; + local adminToken = self["admin-token"]; + local region = self.region; + local replicationFactor = self["replication-factor"]; + local metaSize = self["meta-size"]; + local dataSize = self["data-size"]; + + // Garage daemon configuration file - TOML format + local garage_conf = ||| + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + + db_engine = "lmdb" + + replication_factor = %s + + compression_level = 1 + + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "[::]:3901" + rpc_secret = "%s" + + [s3_api] + s3_region = "%s" + api_bind_addr = "[::]:3900" + root_domain = ".s3.garage.local" + + [s3_web] + bind_addr = "[::]:3902" + root_domain = ".web.garage.local" + index = "index.html" + + [k2v_api] + api_bind_addr = "[::]:3904" + + [admin] + api_bind_addr = "[::]:3903" + admin_token = "%s" + ||| % [replicationFactor, rpcSecret, region, adminToken]; + + // Config volume - contains the rendered garage.toml + local cfgVol = engine.configVolume( + "garage-cfg", "garage", + { + "garage.toml": garage_conf, + } + ); + + // Volumes - Garage stores metadata and data separately + local vol_meta = engine.volume("garage-meta").with_size(metaSize); + local vol_data = engine.volume("garage-data").with_size(dataSize); + + // Main Garage daemon container + local garage_container = + engine.container("garage") + .with_image(images.garage) + .with_command([ + "/garage", "-c", "/etc/garage/garage.toml", "server" + ]) + .with_environment({ + RUST_LOG: "garage=info", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3900, 3900, "s3-api") + .with_port(3901, 3901, "rpc") + .with_port(3902, 3902, "web") + .with_port(3903, 3903, "admin") + .with_port(3904, 3904, "k2v") + .with_volume_mount(cfgVol, "/etc/garage/") + .with_volume_mount(vol_meta, "/var/lib/garage/meta") + .with_volume_mount(vol_data, "/var/lib/garage/data"); + + // Init container - configures cluster layout and creates S3 credentials + // IMPORTANT: This container exits with code 0 after completion + // Orchestrator must NOT restart it (use K8s Job or Compose restart: "no") + // Uses Alpine base image since garage container has no shell + local init_container = + engine.container("garage-init") + .with_image("docker.io/alpine:3.23.2") + .with_environment({ + GARAGE_ACCESS_KEY: accessKey, + GARAGE_SECRET_KEY: secretKey, + GARAGE_REGION: region, + GARAGE_ADMIN_TOKEN: adminToken, + GARAGE_RPC_SECRET: rpcSecret, + GARAGE_DATA_SIZE: dataSize, + }) + .with_limits("0.5", "256M") + .with_reservations("0.25", "128M") + .with_volume_mount(cfgVol, "/etc/garage/") + .with_command([ + "sh", "-c", ||| + set -e + + # Install required tools + echo "Installing curl, jq and downloading garage CLI..." + apk add --no-cache curl jq + + # Download garage binary (v2.1.0) for remote management + curl -fsSL "https://garagehq.deuxfleurs.fr/_releases/v2.1.0/x86_64-unknown-linux-musl/garage" \ + -o /usr/local/bin/garage + chmod +x /usr/local/bin/garage + + echo "Waiting for Garage daemon to be ready..." + MAX_ATTEMPTS=60 + ATTEMPT=0 + # Wait for /health to respond (even 503 is fine - means daemon is up) + until curl -s http://garage:3903/health >/dev/null 2>&1; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: Garage failed to become ready after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: Garage not ready, retrying in 2s..." + sleep 2 + done + echo "Garage daemon is ready." + + # Get the node ID via v2 Admin API + echo "Getting Garage node ID via Admin API..." + curl -s -H "Authorization: Bearer ${GARAGE_ADMIN_TOKEN}" \ + "http://garage:3903/v2/GetNodeInfo?node=self" > /tmp/garage-node-info.json + + # Extract node ID from response (the key in success map is the node ID) + NODE_ID=$(jq -r '.success | to_entries[0].value.nodeId' /tmp/garage-node-info.json) + echo "Node ID: ${NODE_ID}" + + if [ -z "$NODE_ID" ] || [ "$NODE_ID" = "null" ]; then + echo "ERROR: Failed to retrieve node ID" + exit 1 + fi + + # ===== LAYOUT MANAGEMENT VIA REMOTE RPC ===== + # Use garage CLI with -h and -s flags to connect remotely + # -h requires format: @: + # -s provides the RPC secret + + # Construct full RPC identifier + RPC_HOST="${NODE_ID}@garage:3901" + echo "RPC Host: ${RPC_HOST}" + + # Check current layout to see if node is already assigned (idempotent) + echo "Checking current cluster layout..." + LAYOUT_OUTPUT=$(garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" layout show 2>&1) + + # Check if node already has a role assigned + # Layout output shows abbreviated node ID (first 16 chars) + NODE_ID_SHORT="${NODE_ID:0:16}" + if echo "$LAYOUT_OUTPUT" | grep -q "$NODE_ID_SHORT"; then + echo "Node ${NODE_ID_SHORT}... already assigned in layout, skipping." + else + echo "Assigning node to cluster layout..." + # Assign node to zone dc1 with configured capacity + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + layout assign ${NODE_ID} -z dc1 -c ${GARAGE_DATA_SIZE} + + echo "Applying layout configuration..." + # Get current staged version and apply + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + layout apply --version 1 + + echo "Layout configured successfully." + # Wait for layout to stabilize + sleep 5 + fi + + # ===== KEY MANAGEMENT VIA REMOTE RPC ===== + + # Check if key already exists (idempotent) + # GARAGE_ACCESS_KEY is already a valid Garage Key ID (GK + 24 hex chars) + if garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" key info "${GARAGE_ACCESS_KEY}" >/dev/null 2>&1; then + echo "Access key ${GARAGE_ACCESS_KEY} already exists, skipping creation." + else + echo "Importing S3 access key: ${GARAGE_ACCESS_KEY}" + + # Import key with the provided credentials (both already in valid Garage format) + # GARAGE_ACCESS_KEY = Key ID (GK + 24 hex chars) + # GARAGE_SECRET_KEY = Secret (64 hex chars) + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + key import "${GARAGE_ACCESS_KEY}" "${GARAGE_SECRET_KEY}" --yes + + echo "Access key imported successfully." + fi + + # Grant permissions to the key + echo "Granting create-bucket permission to key..." + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + key allow --create-bucket "${GARAGE_ACCESS_KEY}" + + echo "" + echo "Garage initialization complete!" + echo "S3 Endpoint: http://garage:3900" + echo "Region: ${GARAGE_REGION}" + exit 0 + |||, + ]); + + // Container sets + local garage_containerSet = engine.containers("garage", [garage_container]); + local init_containerSet = engine.containers("garage-init", [init_container]); + + // Service - expose Garage ports + local garage_service = + engine.service(garage_containerSet) + .with_port(3900, 3900, "s3-api") + .with_port(3901, 3901, "rpc") + .with_port(3902, 3902, "web") + .with_port(3903, 3903, "admin") + .with_port(3904, 3904, "k2v"); + + engine.resources([ + cfgVol, + vol_meta, + vol_data, + garage_containerSet, + init_containerSet, + garage_service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/memgraph.jsonnet new file mode 100644 index 00000000..eeed2e4e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/memgraph.jsonnet @@ -0,0 +1,70 @@ +local images = import "values/images.jsonnet"; + +{ + + "memgraph" +: { + + create:: function(engine) + + local vol = engine.volume("memgraph").with_size("20G"); + + local container = + engine.container("memgraph") + .with_image(images.memgraph_mage) + .with_environment({ + MEMGRAPH: "--storage-properties-on-edges=true --storage-enable-edges-metadata=true" + }) + .with_limits("1.0", "1000M") + .with_reservations("0.5", "1000M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/var/lib/memgraph"); + + local containerSet = engine.containers( + "memgraph", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + "memgraph-lab" +: { + + create:: function(engine) + + local container = + engine.container("lab") + .with_image(images.memgraph_lab) + .with_environment({ + QUICK_CONNECT_MG_HOST: "memgraph", + QUICK_CONNECT_MG_PORT: "7687", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3010, 3000, "http"); + + local containerSet = engine.containers( + "lab", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3010, 3010, "http"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/milvus.jsonnet new file mode 100644 index 00000000..5bed3820 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/milvus.jsonnet @@ -0,0 +1,89 @@ +local images = import "values/images.jsonnet"; +local minio = import "backends/minio.jsonnet"; + +minio { + + etcd +: { + + create:: function(engine) + + local vol = engine.volume("etcd").with_size("20G"); + + local container = + engine.container("etcd") + .with_image(images.etcd) + .with_command([ + "etcd", + "-advertise-client-urls=http://127.0.0.1:2379", + "-listen-client-urls", + "http://0.0.0.0:2379", + "--data-dir", + "/etcd", + ]) + .with_environment({ + ETCD_AUTO_COMPACTION_MODE: "revision", + ETCD_AUTO_COMPACTION_RETENTION: "1000", + ETCD_QUOTA_BACKEND_BYTES: "4294967296", + ETCD_SNAPSHOT_COUNT: "50000" + }) + .with_limits("1.0", "128M") + .with_reservations("0.25", "128M") + .with_port(2379, 2379, "api") + .with_volume_mount(vol, "/etcd"); + + local containerSet = engine.containers( + "etcd", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(2379, 2379, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + milvus +: { + + create:: function(engine) + + local vol = engine.volume("milvus").with_size("20G"); + + local container = + engine.container("milvus") + .with_image(images.milvus) + .with_command([ + "milvus", "run", "standalone" + ]) + .with_environment({ + ETCD_ENDPOINTS: "etcd:2379", + MINIO_ADDRESS: "minio:9000", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2") + .with_volume_mount(vol, "/var/lib/milvus"); + + local containerSet = engine.containers( + "milvus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/minio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/minio.jsonnet new file mode 100644 index 00000000..b38bb81f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/minio.jsonnet @@ -0,0 +1,48 @@ +local images = import "values/images.jsonnet"; + +{ + + minio +: { + + create:: function(engine) + + local vol = engine.volume("minio-data").with_size("20G"); + + local container = + engine.container("minio") + .with_image(images.minio) + .with_command([ + "minio", + "server", + "/minio_data", + "--console-address", + ":9001", + ]) + .with_environment({ + MINIO_ROOT_USER: "minioadmin", + MINIO_ROOT_PASSWORD: "minioadmin", + }) + .with_limits("0.5", "128M") + .with_reservations("0.25", "128M") + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console") + .with_volume_mount(vol, "/minio_data"); + + local containerSet = engine.containers( + "minio", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/neo4j.jsonnet new file mode 100644 index 00000000..46c61e0f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/neo4j.jsonnet @@ -0,0 +1,46 @@ +local images = import "values/images.jsonnet"; + +{ + + "neo4j" +: { + + create:: function(engine) + + local vol = engine.volume("neo4j").with_size("20G"); + + local container = + engine.container("neo4j") + .with_image(images.neo4j) + .with_environment({ + NEO4J_AUTH: "neo4j/password", + NEO4J_server_memory_pagecache_size: "512m", + NEO4J_server_memory_heap_max__size: "512m", + // NEO4J_server_bolt_listen__address: "0.0.0.0:7687", + // NEO4J_server_default__listen__address: "0.0.0.0", + // NEO4J_server_http_listen__address: "0.0.0.0:7474", + }) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1536M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "neo4j", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/pinecone.jsonnet new file mode 100644 index 00000000..2bef70fb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/pinecone.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "pinecone" +: { + + create:: function(engine) + + local container = + engine.container("pinecone") + .with_image(images.pinecone) + .with_environment({ + + PORT: "5080", + + INDEX_TYPE: "serverless", + + // Dimension is fixed, 384 is right for miniLM + // sentence embedding + DIMENSION: 384, + + METRIC: "cosine" + + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(5080, 5080, "api"); + + local containerSet = engine.containers( + "pinecone", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(5080, 5080, "api"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/qdrant.jsonnet new file mode 100644 index 00000000..2e6761f8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/backends/qdrant.jsonnet @@ -0,0 +1,65 @@ +local images = import "values/images.jsonnet"; + +{ + + "qdrant" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1024M", + "memory-reservation":: "1024M", + + // Mmap settings for low-memory mode (trades latency for memory) + // Set to null to disable, or string value to enable + "memmap-threshold-kb":: null, + "on-disk-payload":: null, + + create:: function(engine) + + // Capture memory settings into locals + local memLimit = self["memory-limit"]; + local memReserv = self["memory-reservation"]; + local mmapThreshold = self["memmap-threshold-kb"]; + local onDiskPayload = self["on-disk-payload"]; + + local vol = engine.volume("qdrant").with_size("20G"); + + // Build environment with optional mmap settings + local baseEnv = {}; + local env = baseEnv + + (if mmapThreshold != null then { + QDRANT__STORAGE__MEMMAP_THRESHOLD_KB: mmapThreshold, + } else {}) + + (if onDiskPayload != null then { + QDRANT__STORAGE__ON_DISK_PAYLOAD: onDiskPayload, + } else {}); + + local container = + engine.container("qdrant") + .with_image(images.qdrant) + .with_limits("1.0", memLimit) + .with_reservations("0.5", memReserv) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2") + .with_volume_mount(vol, "/qdrant/storage") + + (if std.length(env) > 0 then { + environment+: env, + } else {}); + + local containerSet = engine.containers( + "qdrant", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/components.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/components.jsonnet new file mode 100644 index 00000000..03a55208 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/components.jsonnet @@ -0,0 +1,121 @@ +{ + + // Essentials + "trustgraph-base": import "core/trustgraph.jsonnet", + "rev-gateway": import "core/rev-gateway.jsonnet", + "pulsar": import "pulsar/pulsar.jsonnet", + + // LLMs + "azure": import "llm/azure.jsonnet", + "azure-openai": import "llm/azure-openai.jsonnet", + "bedrock": import "llm/bedrock.jsonnet", + "claude": import "llm/claude.jsonnet", + "cohere": import "llm/cohere.jsonnet", + "googleaistudio": import "llm/googleaistudio.jsonnet", + "llamafile": import "llm/llamafile.jsonnet", + "lmstudio": import "llm/lmstudio.jsonnet", + "mistral": import "llm/mistral.jsonnet", + "ollama": import "llm/ollama.jsonnet", + "openai": import "llm/openai.jsonnet", + "vertexai": import "llm/vertexai.jsonnet", + "tgi": import "llm/tgi.jsonnet", + "vllm": import "llm/vllm.jsonnet", + + // Embeddings + "embeddings-ollama": import "embeddings/embeddings-ollama.jsonnet", + "embeddings-hf": import "embeddings/embeddings-hf.jsonnet", + "embeddings-fastembed": import "embeddings/embeddings-fastembed.jsonnet", + + // OCR options + "ocr": import "ocr/ocr.jsonnet", + "mistral-ocr": import "ocr/mistral-ocr.jsonnet", + + // Vector stores + "vector-store-milvus": import "vector-store/milvus.jsonnet", + "vector-store-qdrant": import "vector-store/qdrant.jsonnet", + "vector-store-pinecone": import "vector-store/pinecone.jsonnet", + + // Triples stores + "triple-store-cassandra": import "triple-store/cassandra.jsonnet", + "triple-store-neo4j": import "triple-store/neo4j.jsonnet", + "triple-store-falkordb": import "triple-store/falkordb.jsonnet", + "triple-store-memgraph": import "triple-store/memgraph.jsonnet", + + // Object stores + "row-store-cassandra": import "row-store/cassandra.jsonnet", + + // Observability support + "grafana": import "monitoring/grafana.jsonnet", + "loki": import "monitoring/loki.jsonnet", + + // Pulsar manager is a UI for Pulsar. Uses a LOT of memory + "pulsar-manager": import "pulsar/pulsar-manager.jsonnet", + + "override-recursive-chunker": import "core/chunker-recursive.jsonnet", + + // The prompt manager + "prompt-overrides": import "core/prompt-overrides.jsonnet", + + // Extra MCP services + "ddg-mcp-server": import "mcp/ddg-mcp-server.jsonnet", + + // Does nothing. But, can be a hack to overwrite parameters + "null": {}, + + // Passthrough: returns parameters directly, preserving +: merge syntax + // Also supports JSON-safe routing with prefixed parameters like "cassandra-heap" + "override": { + local route = function(target) + function(prefix, k, v) + local suffix = std.substr(k, std.length(prefix), std.length(k) - std.length(prefix)); + { [target] +: { [suffix]:: v } }, + + local routes = { + "cassandra-": route("cassandra"), + "pulsar-": route("pulsar"), + "qdrant-": route("qdrant"), + "api-gateway-": route("api-gateway"), + "librarian-": route("librarian"), + }, + + with_params:: function(pars) + std.foldl( + function(acc, k) + local matchingPrefixes = [p for p in std.objectFields(routes) if std.startsWith(k, p)]; + if std.length(matchingPrefixes) > 0 then + local prefix = matchingPrefixes[0]; + acc + routes[prefix](prefix, k, pars[k]) + else + acc + { [k]:: pars[k] }, + std.objectFields(pars), + {} + ), + }, + + // Memory profiles + "memory-profile-low": import "profiles/memory-profile-low.jsonnet", + + // Model hosting + + "hosting-intel-battlemage-vllm": + import "model-hosting/intel-battlemage-vllm.jsonnet", + + "hosting-cpu-tgi": + import "model-hosting/cpu-tgi.jsonnet", + + "hosting-intel-xpu-tgi": + import "model-hosting/intel-xpu-tgi.jsonnet", + + "hosting-intel-gaudi-tgi": + import "model-hosting/intel-gaudi-tgi.jsonnet", + + "hosting-intel-xpu-vllm": + import "model-hosting/intel-xpu-vllm.jsonnet", + + "hosting-intel-gaudi-vllm": + import "model-hosting/intel-gaudi-vllm.jsonnet", + + "hosting-nvidia-gpu-vllm": + import "model-hosting/nvidia-gpu-vllm.jsonnet", + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/agent-manager-react.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/agent-manager-react.jsonnet new file mode 100644 index 00000000..954d80a7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/agent-manager-react.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "agent-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("agent-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "agent-manager-react", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "agent-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/chunker-recursive.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/chunker-recursive.jsonnet new file mode 100644 index 00000000..0b924aba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/chunker-recursive.jsonnet @@ -0,0 +1,55 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "chunk-size":: 2000, + "chunk-overlap":: 100, + + "chunker" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + std.toString($["chunk-size"]), + "--chunk-overlap", + std.toString($["chunk-overlap"]), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/configuration.jsonnet new file mode 100644 index 00000000..75afebb9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/configuration.jsonnet @@ -0,0 +1,57 @@ + +// This puts the default configuration together. References many things, +// flow classes, a default flow, token costs, prompts, agent tools + +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "init-trustgraph" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local cfgVol = engine.configVolume( + "trustgraph-cfg", "trustgraph", + { + "config.json": importstr "trustgraph/config.json", + } + ); + + local container = + engine.container("init-trustgraph") + .with_image(images.trustgraph_flow) + .with_command( + [ + "tg-init-trustgraph", + "-p", + url.pulsar_admin, + "--config-file", + "/trustgraph/config.json", + ] + ) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_volume_mount(cfgVol, "/trustgraph/"); + + local containerSet = engine.containers( + "init-trustgraph", [ container ] + ); + + engine.resources([ + cfgVol, + containerSet, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/document-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/document-rag.jsonnet new file mode 100644 index 00000000..e466e3a4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/document-rag.jsonnet @@ -0,0 +1,92 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "document-rag" +: { + + "doc-limit":: 20, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local docLimit = self["doc-limit"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "document-rag", + "-p", + url.pulsar, + "--doc-limit", + std.toString(docLimit), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "document-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "document-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/graph-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/graph-rag.jsonnet new file mode 100644 index 00000000..718441ea --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/graph-rag.jsonnet @@ -0,0 +1,283 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "kg-extract-definitions" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-definitions") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-definitions", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-definitions", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-relationships" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-relationships") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-relationships", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-relationships", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-agent" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-agent") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-agent", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-agent", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-ontology" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "300M", + "memory-reservation":: "300M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-ontology") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-ontology", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-ontology", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-rag" +: { + + concurrency:: 1, + "entity-limit":: 50, + "triple-limit":: 30, + "max-subgraph-size":: 400, + "max-path-length":: 2, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local entityLimit = self["entity-limit"]; + local tripleLimit = self["triple-limit"]; + local maxSubgraphSize = self["max-subgraph-size"]; + local maxPathLength = self["max-path-length"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("graph-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-rag", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--entity-limit", + std.toString(entityLimit), + "--triple-limit", + std.toString(tripleLimit), + "--max-subgraph-size", + std.toString(maxSubgraphSize), + "--max-path-length", + std.toString(maxPathLength), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "graph-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/librarian.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/librarian.jsonnet new file mode 100644 index 00000000..2ecda5a8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/librarian.jsonnet @@ -0,0 +1,58 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local garage = import "backends/garage.jsonnet"; +local cassandra = import "backends/cassandra.jsonnet"; + +{ + + "librarian" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("librarian") + .with_image(images.trustgraph_flow) + .with_command([ + "librarian", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + "--object-store-endpoint", + url.object_store, + "--object-store-access-key", + $.garage["access-key"], + "--object-store-secret-key", + $.garage["secret-key"], + "--object-store-region", + $.garage.region, + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "librarian", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +// Garage and Cassandra are used by the Librarian +} + garage + cassandra + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/mcp-server.jsonnet new file mode 100644 index 00000000..9bf7e3bd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/mcp-server.jsonnet @@ -0,0 +1,54 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "mcp-server" +: { + + port:: 8000, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local port = self.port; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("mcp-server-secret") + .with_env_var("MCP_SERVER_SECRET", "mcp-server-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local container = + engine.container("mcp-server") + .with_image(images.trustgraph_mcp) + .with_command([ + "mcp-server", + "--port", + std.toString(port), + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/prompt-overrides.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/prompt-overrides.jsonnet new file mode 100644 index 00000000..852ec09d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/prompt-overrides.jsonnet @@ -0,0 +1,24 @@ +local default_prompts = import "prompts/default-prompts.jsonnet"; + +{ + + with:: function(key, value) + if (key == "system-template") then + self + { + prompts +:: { + "system-template": value, + } + } + else + self + { + prompts +:: { + templates +:: { + [key] +:: { + prompt: value + } + } + } + }, + +} + default_prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/prompt-template.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/prompt-template.jsonnet new file mode 100644 index 00000000..3cd79d59 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/prompt-template.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "prompt" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("prompt") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "prompt", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "prompt-rag" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("prompt-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--id", + "prompt-rag", + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "prompt-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/rev-gateway.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/rev-gateway.jsonnet new file mode 100644 index 00000000..fbd9f77a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/rev-gateway.jsonnet @@ -0,0 +1,62 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "rev-gateway" +: { + + // Invalid, but at least means the rev-gateway won't connect to anything + // it shouldn't. + token:: "INVALID_TOKEN", + uri:: "wss://127.0.0.1/api/v1/relay?token=" + self.token, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local uri = self.uri; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("rev-gateway-secret") + .with_env_var("REV_GATEWAY_SECRET", "rev-gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "rev-gateway", + "-p", + url.pulsar, + "--websocket-uri", + std.toString(uri), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/structured-data.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/structured-data.jsonnet new file mode 100644 index 00000000..8b23372f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/structured-data.jsonnet @@ -0,0 +1,171 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "nlp-query" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("nlp-query") + .with_image(images.trustgraph_flow) + .with_command([ + "nlp-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "nlp-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-query" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("structured-query") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "structured-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-diag" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "96M", + "memory-reservation":: "96M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("structured-diag") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-diag", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "structured-diag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-objects" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-objects", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/trustgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/trustgraph.jsonnet new file mode 100644 index 00000000..3cf12d26 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/core/trustgraph.jsonnet @@ -0,0 +1,473 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +local config_initialiser = import "configuration.jsonnet"; +local config = import "../runtime-config/trustgraph-config.jsonnet"; +local librarian = import "librarian.jsonnet"; +local mcp_server = import "mcp-server.jsonnet"; +local workbench = import "../ui/workbench-ui.jsonnet"; +local graphrag = import "graph-rag.jsonnet"; +local documentrag = import "document-rag.jsonnet"; +local prompt_template = import "prompt-template.jsonnet"; +local agent_manager = import "agent-manager-react.jsonnet"; +local structured_data = import "structured-data.jsonnet"; +local ddg = import "mcp/ddg-mcp-server.jsonnet"; + +// Helper to create a routing function for a target object +local route = function(target) + function(prefix, k, v) + local suffix = std.substr(k, std.length(prefix), std.length(k) - std.length(prefix)); + { [target] +: { [suffix]:: v } }; + +// Parameter prefix -> target object routing table +local routes = { + "prompt-rag-": route("prompt-rag"), + "prompt-": route("prompt"), + "text-completion-rag-": route("text-completion-rag"), + "text-completion-": route("text-completion"), + "embeddings-": route("embeddings"), + "api-gateway-": route("api-gateway"), + "chunk-": route("chunker"), + "graph-rag-": route("graph-rag"), + "graph-embeddings-": route("graph-embeddings"), + "kg-extract-definitions-": route("kg-extract-definitions"), + "kg-extract-relationships-": route("kg-extract-relationships"), + "kg-extract-agent-": route("kg-extract-agent"), + "kg-extract-ontology-": route("kg-extract-ontology"), + "kg-extract-objects-": route("kg-extract-objects"), + "garage-": route("garage"), + "config-svc-": route("config-svc"), + "pdf-decoder-": route("pdf-decoder"), + "mcp-tool-": route("mcp-tool"), + "mcp-server-": route("mcp-server"), + "metering-rag-": route("metering-rag"), + "metering-": route("metering"), + "kg-store-": route("kg-store"), + "kg-manager-": route("kg-manager"), + "librarian-": route("librarian"), + "agent-manager-": route("agent-manager"), + "document-rag-": route("document-rag"), + "document-embeddings-": route("document-embeddings"), + "rev-gateway-": route("rev-gateway"), + "nlp-query-": route("nlp-query"), + "structured-query-": route("structured-query"), + "structured-diag-": route("structured-diag"), + "init-trustgraph-": route("init-trustgraph"), +}; + +// Find longest matching prefix (most specific first) +local findRoute = function(k) + local prefixes = std.objectFields(routes); + local matching = std.filter(function(p) std.startsWith(k, p), prefixes); + local sorted = std.sort(matching, function(x) -std.length(x)); + if std.length(sorted) > 0 then sorted[0] else null; + +{ + + // Route parameters to appropriate internal objects based on prefix + with:: function(k, v) + local prefix = findRoute(k); + if prefix != null then + self + routes[prefix](prefix, k, v) + else + self + { [k]:: v }, + + "log-level":: "INFO", + + // Base objects with concurrency defaults (LLM/embeddings components merge into these) + "text-completion" +: { concurrency:: 1 }, + "text-completion-rag" +: { concurrency:: 1 }, + embeddings +: { concurrency:: 1 }, + + "api-gateway" +: { + + port:: 8088, + timeout:: 600, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local port = self.port; + local timeout = self.timeout; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("gateway-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "api-gateway", + "-p", + url.pulsar, + "--timeout", + std.toString(timeout), + "--port", + std.toString(port), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "chunker" +: { + + size:: 2000, + overlap:: 50, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local size = self.size; + local overlap = self.overlap; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + std.toString(size), + "--chunk-overlap", + std.toString(overlap), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "config-svc" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local cpuLimit = self["cpu-limit"]; + local cpuReservation = self["cpu-reservation"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("config-svc") + .with_image(images.trustgraph_flow) + .with_command([ + "config-svc", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(cpuLimit, memoryLimit) + .with_reservations(cpuReservation, memoryReservation); + + local containerSet = engine.containers( + "config-svc", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "pdf-decoder" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("pdf-decoder") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-decoder", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "pdf-decoder", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "mcp-tool" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("mcp-tool") + .with_image(images.trustgraph_flow) + .with_command([ + "mcp-tool", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "mcp-tool", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("metering") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "metering", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering-rag" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("metering-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--id", + "metering-rag", + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "metering-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-store" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-store") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-store", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-store", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-manager", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + librarian + mcp_server + workbench + graphrag + + documentrag + prompt_template + agent_manager + structured_data + + config_initialiser + config + + ddg + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/embeddings/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/embeddings/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..37116de0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/embeddings/embeddings-fastembed.jsonnet @@ -0,0 +1,49 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-fastembed.jsonnet"; + +{ + + "fastembed-models":: models, + + "embeddings-models" +:: $["fastembed-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-fastembed", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/embeddings/embeddings-hf.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/embeddings/embeddings-hf.jsonnet new file mode 100644 index 00000000..492f05cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/embeddings/embeddings-hf.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-huggingface.jsonnet"; + +{ + + "huggingface-embeddings-models":: models, + + "embeddings-models" +:: $["huggingface-embeddings-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_hf) + .with_command([ + "embeddings-hf", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/embeddings/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/embeddings/embeddings-ollama.jsonnet new file mode 100644 index 00000000..05cbed72 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/embeddings/embeddings-ollama.jsonnet @@ -0,0 +1,52 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local models = import "parameters/embeddings-ollama.jsonnet"; + +{ + + "ollama-url":: "${OLLAMA_HOST}", + + "ollama-models":: models, + + "embeddings-models" +:: $["ollama-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-r", + $["ollama-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/aks-k8s.jsonnet new file mode 100644 index 00000000..07e050f2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/aks-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "disk.csi.azure.com", + parameters: { + // Standard disks (spinning magnetic), Locally Redundant Storage + // Cheapest, basically + skuName: "Standard_LRS", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/docker-compose.jsonnet new file mode 100644 index 00000000..334b3517 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/docker-compose.jsonnet @@ -0,0 +1,260 @@ +{ + + // Extract resources using the engine + package:: function(patterns) + std.foldl( + function(state, p) state + p.create(self), + std.objectValues(patterns), + {} + ), + + container:: function(name) + { + + local container = self, + + name:: name, + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_group:: function(x) self + + if std.objectHas(container, "group_add") then + { group_add: container.group_add + [x] } + else + { group_add: [x] }, + + with_command:: function(x) self + { + command: + if std.isString(x) then + std.strReplace(x, "$", "$$") + else if std.isArray(x) then + std.map(function(s) std.strReplace(s, "$", "$$"), x) + else + x + }, + + with_entrypoint:: function(x) self + { entrypoint: x }, + + with_runtime:: function(x) self + { runtime: x }, + + with_privileged:: function(x) self + { privileged: x }, + + with_ipc:: function(x) self + { ipc: x }, + + with_capability:: function(x) self + + if std.objectHas(container, "capability") then + { cap_add: container.capability + x } + else + { cap_add: [x], }, + + with_environment:: function(x) self + + if std.objectHas(container, "environment") then + { environment: container.environment + x } + else + { environment: x, }, + + with_device:: function(hdev, cdev) self + + if std.objectHas(container, "devices") then + { devices: container.devices + [ "%s:%s" % [hdev, cdev] ] } + else + { devices: [ "%s:%s" % [hdev, cdev] ], }, + + with_limits:: function(c, m) self + { + deploy +: { resources +: { + limits: { cpus: c, memory: m } + } }, + }, + + with_reservations:: function(c, m) self + { + deploy +: { resources +: { + reservations: { cpus: c, memory: m } + } }, + }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [vol.volid, mnt] + ] + else + [ + "%s:%s" % [vol.volid, mnt] + ] + }, + + with_bind_mount:: + function(src, dest) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [src, dest] + ] + else + [ + "%s:%s" % [src, dest] + ] + }, + + with_port:: + function(src, dest, name) + self + { + ports: + if std.objectHas(container, "ports") then + container.ports + [ "%d:%d" % [src, dest] ] + else + [ "%d:%d" % [src, dest] ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj.with_environment( + { [x]: "${" + x + "}" } + ), + vars.variables, + self + ), + + restart: "on-failure:100", + + add:: function() { + services +: { + [container.name]: container, + } + } + + }, + + internalService:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + with_size:: function(size) self + { size: size }, + + add:: function() { + volumes +: { + [volume.name]: {} + } + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: "./" + dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + variables:: [], + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + }, + + add:: function() { + } + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.foldl( + function(state, c) state + c.add(), + cont.containers, + {} + ), + + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/eks-k8s.jsonnet new file mode 100644 index 00000000..3fc3f035 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/eks-k8s.jsonnet @@ -0,0 +1,46 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "ebs.csi.aws.com", + parameters: { + type: "gp3", + encrypted: "true", + iops: "6000", + throughput: "400", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/gcp-k8s.jsonnet new file mode 100644 index 00000000..71792426 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/gcp-k8s.jsonnet @@ -0,0 +1,44 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "pd.csi.storage.gke.io", + parameters: { + type: "pd-balanced", + "csi.storage.k8s.io/fstype": "ext4", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/k8s.jsonnet new file mode 100644 index 00000000..2067f6ac --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/k8s.jsonnet @@ -0,0 +1,393 @@ +{ + + container:: function(name) + { + + local container = self, + + name: name, + limits: {}, + reservations: {}, + ports: [], + volumes: [], + bindMounts: [], + groups: [], + environment: [], + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_group:: function(x) self + { groups: super.groups + [x] }, + + with_privileged:: function(x) self + { privileged: x }, + + with_command:: function(x) self + { command: x }, + + with_entrypoint:: function(x) self + { entrypoint: x }, + + with_environment:: function(x) self + { + environment: super.environment + [ + { + name: v.key, value: v.value + } + for v in std.objectKeysValues(x) + ], + }, + + with_limits:: function(c, m) self + { limits: { cpu: c, memory: m } }, + + with_reservations:: + function(c, m) self + { reservations: { cpu: c, memory: m } }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: super.volumes + [{ + volume: vol, mount: mnt + }] + }, + + with_bind_mount:: + function(src, dest) + local name = "bind-" + std.strReplace(std.strReplace(src, "/", "-"), ".", "-"); + self + { + bindMounts: super.bindMounts + [{ + name: name, src: src, dest: dest + }] + }, + + with_port:: + function(src, dest, name) self + { + ports: super.ports + [ + { src: src, dest: dest, name : name } + ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj + { + environment: super.environment + [{ + name: x, + valueFrom: { + secretKeyRef: { + name: vars.name, + key: vars.keyMap[x], + } + } + }] + }, + vars.variables, + self + ), + + add:: function() [ + + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: container.name, + namespace: "trustgraph", + labels: { + app: container.name + } + }, + spec: { + replicas: 1, + selector: { + matchLabels: { + app: container.name, + } + }, + template: { + metadata: { + labels: { + app: container.name, + } + }, + spec: { + containers: [ + { + name: container.name, + image: container.image, + + // FIXME: Make everything run as + // root. Needed to get filesystems + // to be accessible. There's a + // better way of doing this? + securityContext: { + runAsUser: 0, + runAsGroup: 0, + } + ( + if std.objectHas(container, "privileged") && container.privileged then + { privileged: true } + else {} + ), + + resources: { + requests: container.reservations, + limits: container.limits + }, + } + ( + if std.length(container.ports) > 0 then + { + ports: [ + { + hostPort: port.src, + containerPort: port.dest, + } + for port in container.ports + ] + } else + {}) + + + (if std.objectHas(container, "entrypoint") then + // Entrypoint is set - use command for entrypoint, args for command + (if std.isString(container.entrypoint) && container.entrypoint == "" then + { command: [] } + else if std.isArray(container.entrypoint) then + { command: container.entrypoint } + else + { command: [container.entrypoint] } + ) + (if std.objectHas(container, "command") then + { args: container.command } + else {}) + else if std.objectHas(container, "command") then + { command: container.command } + else {}) + + + (if std.length(container.environment) > 0 then + { + env: container.environment, + } + else {}) + + + (if std.length(container.volumes) > 0 || std.length(container.bindMounts) > 0 then + { + volumeMounts: [ + { + mountPath: vol.mount, + name: vol.volume.name, + } + for vol in container.volumes + ] + [ + { + mountPath: bm.dest, + name: bm.name, + } + for bm in container.bindMounts + ] + } + + else + {} + ) + ], + volumes: [ + vol.volume.volRef() + for vol in container.volumes + ] + [ + { + name: bm.name, + hostPath: { path: bm.src } + } + for bm in container.bindMounts + ] + } + ( + if std.length(container.groups) > 0 then + { securityContext: { supplementalGroups: container.groups } } + else {} + ) + }, + } + {} + + } + + ] + + }, + + // Just an alias + internalService:: self.service, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + ports: [], + + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + + add:: function() [ + + { + + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + storageClassName: "tg", + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + data: parts + }, + ], + + + volRef:: function() { + name: volume.name, + configMap: { name: volume.name }, + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + variables: [], + keyMap: {}, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + }, + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + keyMap: super.keyMap + { [name]: key }, + }, + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.flattenArrays( + [ c.add() for c in cont.containers ] + ), + + }, + + resources:: function(res) + + std.flattenArrays( + [ c.add() for c in res ] + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/minikube-k8s.jsonnet new file mode 100644 index 00000000..858b17ad --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/minikube-k8s.jsonnet @@ -0,0 +1,115 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList, + + volume:: function(name) + { + local volume = self, + name: name, + with_size:: function(size) self + { size: size }, + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolume", + metadata: { + name: volume.name, + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + capacity: { + storage: volume.size, + }, + persistentVolumeReclaimPolicy: "Delete", + hostPath: { + path: "/data/pv-" + volume.name, + }, + } + }, + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + service:: function(containers) + { + local service = self, + name: containers.name, + ports: [], + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + add:: function() [ + { + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + type: "LoadBalancer", + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/noop.jsonnet new file mode 100644 index 00000000..1f384648 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/noop.jsonnet @@ -0,0 +1,79 @@ +{ + + // Extract resources usnig the engine + package:: function(patterns) {}, + + container:: function(name) { + + with_image:: function(x) self + {}, + + with_user:: function(x) self + {}, + + with_command:: function(x) self + {}, + + with_runtime:: function(x) self + {}, + + with_privileged:: function(x) self + {}, + + with_ipc:: function(x) self + {}, + + with_capability:: function(x) self + {}, + + with_environment:: function(x) self + {}, + + with_device:: function(hdev, cdev) self + {}, + + with_limits:: function(c, m) self + {}, + + with_reservations:: function(c, m) self + {}, + + with_volume_mount:: self + {}, + + with_port:: function(src, dest, name) self + {}, + + with_env_var_secrets:: function(vars) self + {}, + + add:: function() {}, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + service:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + volume:: function(name) { + with_size:: function(size) self + {}, + add:: function() {}, + }, + + configVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + secretVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self + {}, + add:: function() {}, + }, + + containers:: function(name, containers) { + add:: function() {}, + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/ovh-k8s.jsonnet new file mode 100644 index 00000000..15d57c83 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/ovh-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "cinder.csi.openstack.org", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", + parameters: { + availability: "nova", + fsType: "ext4", + type: "high-speed", + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/scw-k8s.jsonnet new file mode 100644 index 00000000..eed23968 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/engine/scw-k8s.jsonnet @@ -0,0 +1,40 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "csi.scaleway.com", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/agent-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/agent-extract.jsonnet new file mode 100644 index 00000000..4c5785c8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/agent-extract.jsonnet @@ -0,0 +1,35 @@ +// Agent-based extraction module +// Uses AI agents for more sophisticated knowledge extraction from text +// Leverages agent tools and reasoning for complex extraction tasks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - internal agent extraction service + "interfaces" +: { + }, + + // No configurable parameters for agent extraction + "parameters" +: { + }, + + // Flow-level processors for agent-based extraction + "flow" +: { + // Agent-based knowledge extraction processor + // Uses AI agents with tools to extract structured knowledge + "kg-extract-agent:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output knowledge triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "agent-request": request("agent:{id}"), // Agent service requests + "agent-response": response("agent:{id}"), // Agent service responses + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/agent.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/agent.jsonnet new file mode 100644 index 00000000..6ed769b6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/agent.jsonnet @@ -0,0 +1,51 @@ +// Agent management module +// Provides AI agent orchestration and tool integration +// Manages agent conversations, tool calls, and response coordination +// Supports MCP tools, GraphRAG, and structured queries + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services (agent requires LLM for reasoning, MCP for tools) +local llm_services = import "llm-services.jsonnet"; +local mcp_service = import "mcp-service.jsonnet"; + +// Merge shared services with agent-specific configuration +llm_services + mcp_service + { + + // External interfaces for agent operations + "interfaces" +: { + "agent": request_response("agent:{id}"), + }, + + // Flow-level processors for agent management + "flow" +: { + // Agent manager orchestrates agent conversations and tool usage + "agent-manager:{id}": { + // Agent communication channels + request: request("agent:{id}"), + next: request("agent:{id}"), + response: response("agent:{id}"), + + // LLM and prompt services + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + + // Tool integrations + "mcp-tool-request": request("mcp-tool:{id}"), + "mcp-tool-response": response("mcp-tool:{id}"), + "graph-rag-request": request("graph-rag:{id}"), + "graph-rag-response": response("graph-rag:{id}"), + "structured-query-request": request("structured-query:{id}"), + "structured-query-response": response("structured-query:{id}"), + }, + }, + + // Blueprint-level processors for agent-related services + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/document-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/document-store.jsonnet new file mode 100644 index 00000000..676765b2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/document-store.jsonnet @@ -0,0 +1,56 @@ +// Document store module +// Infrastructure for document-based RAG using chunk embeddings +// Handles document embedding storage, retrieval, and question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with document store configuration +llm_services + embeddings_service + { + + // External interfaces for document store + "interfaces" +: { + // Document embedding storage and retrieval + "document-embeddings-store": flow("document-embeddings-store:{id}"), + "document-rag": request_response("document-rag:{id}"), + "document-embeddings": request_response("document-embeddings:{id}"), + }, + + // Flow-level processors for document embedding and storage + "flow" +: { + "document-embeddings:{id}": { + input: flow("chunk-load:{id}"), + output: flow("document-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "de-write:{id}": { + input: flow("document-embeddings-store:{id}"), + }, + "document-rag:{id}": { + request: request("document-rag:{id}"), + response: response("document-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "document-embeddings-request": request("document-embeddings:{id}"), + "document-embeddings-response": response("document-embeddings:{id}"), + }, + "de-query:{id}": { + request: request("document-embeddings:{id}"), + response: response("document-embeddings:{id}"), + }, + }, + + // Blueprint-level processors for document RAG operations + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/embeddings-service.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/embeddings-service.jsonnet new file mode 100644 index 00000000..1537e6c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/embeddings-service.jsonnet @@ -0,0 +1,30 @@ +// Shared embeddings service module +// Provides vector embedding generation for text +// Import this module in any flow that requires embeddings + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // Interfaces exposed by embeddings service + "interfaces" +: { + "embeddings": request_response("embeddings:{id}"), + }, + + "parameters" +: { + }, + + // Flow-level processor for embeddings + "flow" +: { + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/flow-blueprints.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/flow-blueprints.jsonnet new file mode 100644 index 00000000..4282fa23 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/flow-blueprints.jsonnet @@ -0,0 +1,48 @@ +// TrustGraph Flow Blueprints Configuration +// +// RAG Modes (4 types): +// - Document RAG: Uses document chunk embeddings +// - Graph RAG: Extracts definitions + relationships to graph +// - Ontology RAG: Extracts using ontology definitions to graph (mutually exclusive with Graph RAG) +// - Structured RAG: Extracts objects to object store +// +// Module structure: +// - *-store: Storage and query infrastructure +// - *-extract: Extraction methods + +// Import all the modular flow components +local graph_store = import "graph-store.jsonnet"; +local document_store = import "document-store.jsonnet"; +local structured_store = import "structured-store.jsonnet"; +local graphrag_extract = import "graphrag-extract.jsonnet"; +local ontorag_extract = import "ontorag-extract.jsonnet"; +local structured_extract = import "structured-extract.jsonnet"; +local agent = import "agent.jsonnet"; +local load = import "load.jsonnet"; +local kgcore = import "kgcore.jsonnet"; + +{ + + // Full system: Graph RAG + Document RAG + knowledge cores + "everything": { + description: "Graph RAG + Document RAG + knowledge cores", + tags: ["document-rag", "graph-rag", "kgcore"], + } + + graph_store + document_store + agent + load + + graphrag_extract + kgcore, + + // Structured RAG only + "structured": { + description: "Structured data extraction and querying", + tags: ["structured"], + } + + structured_store + structured_extract + agent + load, + + // Ontology RAG + knowledge cores + "ontology": { + description: "Ontology RAG + knowledge cores", + tags: ["onto-rag", "kgcore"], + } + + graph_store + ontorag_extract + agent + load + kgcore, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/graph-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/graph-store.jsonnet new file mode 100644 index 00000000..c6f16083 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/graph-store.jsonnet @@ -0,0 +1,70 @@ +// Graph store module +// Shared infrastructure for graph-based RAG (used by both GraphRAG and OntologyRAG) +// Handles knowledge graph storage, embeddings, and graph-based question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with graph store configuration +llm_services + embeddings_service + { + + // External interfaces exposed by the graph store + "interfaces" +: { + // Data ingestion interfaces for graph construction + "entity-contexts-load": flow("entity-contexts-load:{id}"), + "triples-store": flow("triples-store:{id}"), + "graph-embeddings-store": flow("graph-embeddings-store:{id}"), + + // Query interfaces for graph-based operations + "graph-rag": request_response("graph-rag:{id}"), + "triples": request_response("triples:{id}"), + "graph-embeddings": request_response("graph-embeddings:{id}"), + }, + + // Flow-level processors - handle data streams for a specific flow instance + "flow" +: { + "graph-embeddings:{id}": { + input: flow("entity-contexts-load:{id}"), + output: flow("graph-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "triples-write:{id}": { + input: flow("triples-store:{id}"), + }, + "ge-write:{id}": { + input: flow("graph-embeddings-store:{id}"), + }, + "graph-rag:{id}": { + request: request("graph-rag:{id}"), + response: response("graph-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "graph-embeddings-request": request("graph-embeddings:{id}"), + "graph-embeddings-response": response("graph-embeddings:{id}"), + "triples-request": request("triples:{id}"), + "triples-response": response("triples:{id}"), + }, + "triples-query:{id}": { + request: request("triples:{id}"), + response: response("triples:{id}"), + }, + "ge-query:{id}": { + request: request("graph-embeddings:{id}"), + response: response("graph-embeddings:{id}"), + }, + }, + + // Blueprint-level processors - shared across all flow instances of this blueprint + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/graphrag-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/graphrag-extract.jsonnet new file mode 100644 index 00000000..6bcbca14 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/graphrag-extract.jsonnet @@ -0,0 +1,44 @@ +// GraphRAG extraction module +// Extraction method for GraphRAG - extracts definitions and relationships +// Mutually exclusive with OntologyRAG extraction (both write to graph store) + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts entity definitions from text chunks + // Identifies and defines key entities mentioned in the text + "kg-extract-definitions:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output definition triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition extraction prompts + "prompt-response": response("prompt:{id}"), + }, + + // Extracts relationships between entities + // Identifies how entities are connected and interact + "kg-extract-relationships:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output relationship triples + "prompt-request": request("prompt:{id}"), // Relationship extraction prompts + "prompt-response": response("prompt:{id}"), + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/helpers.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/helpers.jsonnet new file mode 100644 index 00000000..eabb3bf4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/helpers.jsonnet @@ -0,0 +1,29 @@ +// Helper functions for flow configuration +// Provides utility functions for constructing flow, request, and response URIs +// used throughout the TrustGraph flow configuration system + +// Creates a persistent flow URI for data streams +// Persistent flows retain messages until consumed +local flow(x) = "persistent://tg/flow/" + x; + +// Creates a non-persistent request URI for request-response patterns +// Non-persistent means messages are not retained if no consumer is present +local request(x) = "non-persistent://tg/request/" + x; + +// Creates a non-persistent response URI for request-response patterns +local response(x) = "non-persistent://tg/response/" + x; + +// Creates a request-response pair for bidirectional communication +// Returns an object with both request and response URIs +local request_response(x) = { + request: request(x), + response: response(x), +}; + +// Export all helper functions for use in other modules +{ + flow: flow, + request: request, + response: response, + request_response: request_response, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/kgcore.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/kgcore.jsonnet new file mode 100644 index 00000000..bfc01ada --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/kgcore.jsonnet @@ -0,0 +1,31 @@ +// Knowledge Graph Core storage module +// Handles persistent storage of knowledge graph data +// Consolidates triples and graph embeddings into permanent storage +// Creates the core knowledge base for long-term use + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; + +{ + // No external interfaces - internal storage service + "interfaces" +: { + }, + + // No configurable parameters for core storage + "parameters" +: { + }, + + // Flow-level processors for knowledge graph storage + "flow" +: { + // Knowledge graph store consolidates extracted knowledge + // Takes processed triples and embeddings and stores them permanently + "kg-store:{id}": { + "triples-input": flow("triples-store:{id}"), // Input RDF triples stream + "graph-embeddings-input": flow("graph-embeddings-store:{id}"), // Input graph embeddings + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/llm-parameters.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/llm-parameters.jsonnet new file mode 100644 index 00000000..a99f72ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/llm-parameters.jsonnet @@ -0,0 +1,59 @@ +{ + + // LLM model selection for normal LLM + "llm-model": { + "type": "llm-model", + "description": "LLM model", + "order": 1, + "advanced": false, + }, + + // LLM model for RAG operations + "llm-rag-model": { + "type": "llm-model", + "description": "LLM model for RAG", + "order": 2, + "advanced": true, + "controlled-by": "llm-model", + }, + + // LLM model selection for normal LLM + "llm-temperature": { + "type": "llm-temperature", + "description": "LLM temperature", + "order": 3, + "advanced": true, + }, + + // LLM model selection for normal LLM + "llm-rag-temperature": { + "type": "llm-temperature", + "description": "LLM temperature for RAG", + "order": 4, + "advanced": true, + }, + + "embeddings-model": { + "type": "embeddings-model", + "description": "Embeddings model", + "order": 5, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-size": { + "type": "chunk-size", + "description": "Chunk size", + "order": 6, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-overlap": { + "type": "chunk-overlap", + "description": "Chunk overlap", + "order": 7, + "advanced": true, + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/llm-services.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/llm-services.jsonnet new file mode 100644 index 00000000..ee6e09ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/llm-services.jsonnet @@ -0,0 +1,66 @@ +// Shared LLM services module +// Provides text completion, prompt processing, and metering services +// Import this module in any flow that requires LLM functionality + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // Interfaces exposed by LLM services + "interfaces" +: { + "prompt": request_response("prompt:{id}"), + "text-completion": request_response("text-completion:{id}"), + }, + + // LLM configuration parameters + "parameters" +: llm_parameters, + + // Flow-level processors for LLM services + "flow" +: { + // Primary text completion service + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + + // RAG-specific text completion (may use different model) + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + + // Prompt processing service + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + + // RAG-specific prompt processing + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + + // Usage metering for primary completion + "metering:{id}": { + input: response("text-completion:{id}"), + }, + + // Usage metering for RAG completion + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/load.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/load.jsonnet new file mode 100644 index 00000000..6c440792 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/load.jsonnet @@ -0,0 +1,43 @@ +// Document loading and preprocessing module +// Handles document ingestion, format conversion, and chunking +// Converts PDFs to text and splits documents into processable chunks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services (load requires embeddings for chunk processing) +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with load-specific configuration +embeddings_service + { + + // External interfaces for document loading + "interfaces" +: { + "document-load": flow("document-load:{id}"), + "text-load": flow("text-document-load:{id}"), + }, + + // Flow-level processors for document preprocessing + "flow" +: { + // PDF decoder converts PDF documents to text + "pdf-decoder:{id}": { + input: flow("document-load:{id}"), + output: flow("text-document-load:{id}"), + }, + + // Chunker splits documents into smaller, processable pieces + "chunker:{id}": { + input: flow("text-document-load:{id}"), + output: flow("chunk-load:{id}"), + "chunk-size": "{chunk-size}", + "chunk-overlap": "{chunk-overlap}", + }, + }, + + // Blueprint-level processors for document loading services + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/mcp-service.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/mcp-service.jsonnet new file mode 100644 index 00000000..5d599a67 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/mcp-service.jsonnet @@ -0,0 +1,31 @@ +// Shared MCP (Model Context Protocol) tool service module +// Provides MCP tool execution capabilities for agents +// Import this module in any flow that requires MCP tool integration + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // Interfaces exposed by MCP service + "interfaces" +: { + "mcp-tool": request_response("mcp-tool:{id}"), + }, + + "parameters" +: { + }, + + // Flow-level processor for MCP tool execution + "flow" +: { + "mcp-tool:{id}": { + request: request("mcp-tool:{id}"), + response: response("mcp-tool:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/ontorag-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/ontorag-extract.jsonnet new file mode 100644 index 00000000..e5521d8a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/ontorag-extract.jsonnet @@ -0,0 +1,39 @@ +// OntologyRAG extraction module +// Extraction method for OntologyRAG - extracts using ontology definitions +// Mutually exclusive with GraphRAG extraction (both write to graph store) + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts using ontology definitions + "kg-extract-ontology:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition + // extraction prompts + "prompt-response": response("prompt:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/structured-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/structured-extract.jsonnet new file mode 100644 index 00000000..599a0c31 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/structured-extract.jsonnet @@ -0,0 +1,30 @@ +// Structured RAG extraction module +// Extracts structured objects from text chunks +// Outputs to objects-store for structured data querying + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + "interfaces" +: { + }, + + "parameters" +: { + }, + + // Flow-level processor for structured object extraction + "flow" +: { + "kg-extract-objects:{id}": { + input: flow("chunk-load:{id}"), + output: flow("objects-store:{id}"), + "entity-contexts": flow("entity-contexts-load:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/structured-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/structured-store.jsonnet new file mode 100644 index 00000000..e6dd1844 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/flows/structured-store.jsonnet @@ -0,0 +1,63 @@ +// Structured store module +// Shared infrastructure for structured data RAG +// Handles object storage, retrieval, and NLP query capabilities + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with structured store configuration +llm_services + embeddings_service + { + + // External interfaces for structured store + "interfaces" +: { + // Object storage and querying + "objects-store": flow("objects-store:{id}"), + "objects": request_response("objects:{id}"), + + // Query interfaces + "nlp-query": request_response("nlp-query:{id}"), + "structured-query": request_response("structured-query:{id}"), + "structured-diag": request_response("structured-diag:{id}"), + }, + + // Flow-level processors for structured storage and query + "flow" +: { + "objects-write:{id}": { + input: flow("objects-store:{id}"), + }, + "objects-query:{id}": { + request: request("objects:{id}"), + response: response("objects:{id}"), + }, + "nlp-query:{id}": { + request: request("nlp-query:{id}"), + response: response("nlp-query:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + }, + "structured-query:{id}": { + request: request("structured-query:{id}"), + response: response("structured-query:{id}"), + "nlp-query-request": request("nlp-query:{id}"), + "nlp-query-response": response("nlp-query:{id}"), + "objects-query-request": request("objects:{id}"), + "objects-query-response": response("objects:{id}"), + }, + "structured-diag:{id}": { + request: request("structured-diag:{id}"), + response: response("structured-diag:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/azure-openai.jsonnet new file mode 100644 index 00000000..bee0fe47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/azure-openai.jsonnet @@ -0,0 +1,112 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure-openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-openai-" + key]:: value, + }, + +// Strategy is to specify the model with the AZURE_MODEL environment +// variable. This isn't something that can just be specified dynamically, +// it has to match what was provisioned in Azure. + + "azure-openai-max-output-tokens":: 4192, + "azure-openai-temperature":: 0.0, + "azure-openai-models":: models, + + "llm-models" +:: $["azure-openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/azure.jsonnet new file mode 100644 index 00000000..f4db7500 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/azure.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-" + key]:: value, + }, + + "azure-max-output-tokens":: 4096, + "azure-temperature":: 0.0, + "azure-models":: models, + + "llm-models" +:: $["azure-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/bedrock.jsonnet new file mode 100644 index 00000000..0db1475b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/bedrock.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/bedrock.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["bedrock-" + key]:: value, + }, + + "bedrock-max-output-tokens":: 4096, + "bedrock-temperature":: 0.0, + "bedrock-models":: models, + + "llm-models" +:: $["bedrock-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/claude.jsonnet new file mode 100644 index 00000000..f3125e96 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/claude.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/claude.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["claude-" + key]:: value, + }, + + "claude-max-output-tokens":: 4096, + "claude-temperature":: 0.0, + "claude-models":: models, + + "llm-models" +:: $["claude-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/cohere.jsonnet new file mode 100644 index 00000000..6517f7a0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/cohere.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/cohere.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["cohere-" + key]:: value, + }, + + "cohere-temperature":: 0.0, + "cohere-models":: models, + + "llm-models" +:: $["cohere-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/googleaistudio.jsonnet new file mode 100644 index 00000000..f7e8d578 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/googleaistudio.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/googleaistudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["googleaistudio-" + key]:: value, + }, + + "googleaistudio-max-output-tokens":: 4096, + "googleaistudio-temperature":: 0.0, + "googleaistudio-models":: models, + + "llm-models" +:: $["googleaistudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/llamafile.jsonnet new file mode 100644 index 00000000..f3bee2cc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/llamafile.jsonnet @@ -0,0 +1,94 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/slm.jsonnet"; +local models = import "parameters/llamafile.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["llamafile-" + key]:: value, + }, + + "llamafile-models":: models, + + "llm-models" +:: $["llamafile-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/lmstudio.jsonnet new file mode 100644 index 00000000..4cf78caa --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/lmstudio.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/lmstudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["lmstudio-" + key]:: value, + }, + + "lmstudio-max-output-tokens":: 4096, + "lmstudio-temperature":: 0.0, + "lmstudio-models":: models, + + "llm-models" +:: $["lmstudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/mistral.jsonnet new file mode 100644 index 00000000..2fb8c562 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/mistral.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/mistral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "mistral-max-output-tokens":: 4096, + "mistral-temperature":: 0.0, + "mistral-models":: models, + + "llm-models" +:: $["mistral-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/ollama.jsonnet new file mode 100644 index 00000000..6143f5cc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/ollama.jsonnet @@ -0,0 +1,102 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/ollama.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["ollama-" + key]:: value, + }, + + "ollama-models":: models, + + "llm-models" +:: $["ollama-models"], + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/openai.jsonnet new file mode 100644 index 00000000..c3d482fd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/openai.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["openai-" + key]:: value, + }, + + "openai-max-output-tokens":: 4096, + "openai-temperature":: 0.0, + "openai-models":: models, + + "llm-models" +:: $["openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/tgi.jsonnet new file mode 100644 index 00000000..91588dff --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/tgi.jsonnet @@ -0,0 +1,108 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-" + key]:: value, + }, + + "tgi-max-output-tokens":: 1024, + "tgi-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/vertexai.jsonnet new file mode 100644 index 00000000..cb8c141f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/vertexai.jsonnet @@ -0,0 +1,120 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vertexai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vertexai-" + key]:: value, + }, + + "vertexai-private-key":: "/vertexai/private.json", + "vertexai-region":: "us-central1", + "vertexai-max-output-tokens":: 4096, + "vertexai-temperature":: 0.0, + "vertexai-models":: models, + + "llm-models" +:: $["vertexai-models"], + + "text-completion" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion-rag") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/vllm.jsonnet new file mode 100644 index 00000000..8b846bbf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/llm/vllm.jsonnet @@ -0,0 +1,113 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vllm.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-" + key]:: value, + }, + + "vllm-models":: models, + + "llm-models" +:: $["vllm-models"], + + "vllm-max-output-tokens":: 1024, + "vllm-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/mcp/ddg-mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/mcp/ddg-mcp-server.jsonnet new file mode 100644 index 00000000..04176f15 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/mcp/ddg-mcp-server.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "ddg-mcp-server-port":: 9870, + + "ddg-mcp-server" +: { + + create:: function(engine) + + local port = $["ddg-mcp-server-port"]; + + local container = + engine.container("ddg-mcp-server") + .with_image(images["ddg-mcp-server"]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "ddg-mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + mcp +:: { + "duckduckgo": { + "remote-name": "search", + local port = $["ddg-mcp-server-port"], + local url = "http://ddg-mcp-server:%s/mcp" % port, + "url": url, + } + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/cpu-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/cpu-tgi.jsonnet new file mode 100644 index 00000000..04b61566 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/cpu-tgi.jsonnet @@ -0,0 +1,68 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + "tgi-service-storage":: "20G", + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-cpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--cuda-graphs", + "0", + ]) + .with_environment({ + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-battlemage-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-battlemage-vllm.jsonnet new file mode 100644 index 00000000..7646398c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-battlemage-vllm.jsonnet @@ -0,0 +1,98 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-Nemo-Instruct-2407", + "vllm-service-cpus":: "32.0", + "vllm-service-memory":: "48G", + "vllm-service-storage":: "48G", + "vllm-service-tokenizer-mode":: "mistral", + "vllm-service-datatype":: "float16", + "vllm-service-quantization":: "woq_int4", + "vllm-service-hf-token":: null, + "vllm-service-max-model-len":: 8192, + "vllm-service-max-num-seqs":: 16, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-battlemage"]) + .with_entrypoint("") // Clear default entrypoint + .with_command([ + "python", + "-m", + "ipex_llm.vllm.xpu.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--device", + "xpu", + "--dtype", + $["vllm-service-datatype"], + "--enforce-eager", + "--max-model-len", + std.toString($["vllm-service-max-model-len"]), + "--max-num-seqs", + std.toString($["vllm-service-max-num-seqs"]), + "--load-in-low-bit", + $["vllm-service-quantization"], + "--trust-remote-code", + "--tokenizer-mode", + $["vllm-service-tokenizer-mode"], + "--disable-sliding-window", + ]) + .with_environment({ + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS: "1", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-gaudi-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-gaudi-tgi.jsonnet new file mode 100644 index 00000000..5af36b78 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-gaudi-tgi.jsonnet @@ -0,0 +1,96 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "meta-llama/Llama-3.3-70B-Instruct", + "tgi-service-cpus":: "64.0", + "tgi-service-memory":: "64G", + "tgi-service-storage":: "50G", + "tgi-service-num-shard":: 8, + "tgi-service-max-input-tokens":: 4096, + "tgi-service-max-total-tokens":: 4096, + "tgi-service-max-batch-size":: 128, + "tgi-service-max-concurrent-requests":: 512, + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-gaudi"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--sharded", + "true", + "--num-shard", + std.toString($["tgi-service-num-shard"]), + "--max-input-tokens", + std.toString($["tgi-service-max-input-tokens"]), + "--max-total-tokens", + std.toString($["tgi-service-max-total-tokens"]), + "--max-batch-size", + std.toString($["tgi-service-max-batch-size"]), + "--max-waiting-tokens", + "7", + "--max-concurrent-requests", + std.toString($["tgi-service-max-concurrent-requests"]), + "--cuda-graphs", + "0", + ]) + .with_runtime("habana") + .with_environment({ + HABANA_VISIBLE_DEVICES: "all", + OMPI_MCA_btl_vader_single_copy_mechanism: "none", + ENABLE_HPU_GRAPH: "true", + LIMIT_HPU_GRAPH: "true", + USE_FLASH_ATTENTION: "true", + FLASH_ATTENTION_RECOMPUTE: "true", + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-gaudi-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-gaudi-vllm.jsonnet new file mode 100644 index 00000000..81d31c14 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-gaudi-vllm.jsonnet @@ -0,0 +1,76 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "64.0", + "vllm-service-memory":: "64G", + "vllm-service-storage":: "50G", + "vllm-service-tensor-parallel-size":: 8, + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-gaudi"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--tensor-parallel-size", + std.toString($["vllm-service-tensor-parallel-size"]), + ]) + .with_runtime("habana") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + HABANA_VISIBLE_DEVICES: "all", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-xpu-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-xpu-tgi.jsonnet new file mode 100644 index 00000000..bc806874 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-xpu-tgi.jsonnet @@ -0,0 +1,75 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + "tgi-service-storage":: "20G", + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-intel-xpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--cuda-graphs", + "0", + ]) + .with_environment({ + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_group("video") + .with_group("render") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-xpu-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-xpu-vllm.jsonnet new file mode 100644 index 00000000..f6e82905 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/intel-xpu-vllm.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "8.0", + "vllm-service-memory":: "16G", + "vllm-service-storage":: "20G", + "vllm-service-datatype":: "float16", + "vllm-service-max-model-len":: 4096, + "vllm-service-max-num-seqs":: 16, + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-xpu"]) + .with_command([ + "python", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--device", + "xpu", + "--dtype", + $["vllm-service-datatype"], + "--enforce-eager", + "--max-model-len", + std.toString($["vllm-service-max-model-len"]), + "--max-num-seqs", + std.toString($["vllm-service-max-num-seqs"]), + "--block-size", + "64", + "--gpu-memory-util", + "0.85", + "--trust-remote-code", + "--disable-sliding-window", + ]) + .with_environment({ + VLLM_USE_V1: "1", + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_group("video") + .with_group("render") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/nvidia-gpu-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/nvidia-gpu-vllm.jsonnet new file mode 100644 index 00000000..d04876c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/model-hosting/nvidia-gpu-vllm.jsonnet @@ -0,0 +1,72 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-7B-Instruct-v0.3", + "vllm-service-cpus":: "0.5", + "vllm-service-memory":: "1G", + "vllm-service-storage":: "50G", + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-nvidia"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + ]) + .with_runtime("nvidia") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/monitoring/grafana.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/monitoring/grafana.jsonnet new file mode 100644 index 00000000..dd0dddd7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/monitoring/grafana.jsonnet @@ -0,0 +1,124 @@ +local images = import "values/images.jsonnet"; +local loki = import "loki.jsonnet"; + +{ + + "prometheus" +: { + + create:: function(engine) + + local vol = engine.volume("prometheus-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "prometheus-cfg", "prometheus", + { + "prometheus.yml": importstr "prometheus/prometheus.yml", + } + ); + + local container = + engine.container("prometheus") + .with_image(images.prometheus) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M") + .with_port(9090, 9090, "http") + .with_volume_mount(cfgVol, "/etc/prometheus/") + .with_volume_mount(vol, "/prometheus"); + + local containerSet = engine.containers( + "prometheus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9090, 9090, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + + "grafana" +: { + + create:: function(engine) + + local vol = engine.volume("grafana-storage").with_size("20G"); + + local provDashVol = engine.configVolume( + "prov-dash", "grafana/provisioning/", + { + "dashboard.yml": + importstr "grafana/provisioning/dashboard.yml", + } + + ); + + local provDataVol = engine.configVolume( + "prov-data", "grafana/provisioning/", + { + "datasource.yml": + importstr "grafana/provisioning/datasource.yml", + } + + ); + + local dashVol = engine.configVolume( + "dashboards", "grafana/dashboards/", + { + "overview-dashboard.json": + importstr "grafana/dashboards/overview-dashboard.json", + "log-dashboard.json": + importstr "grafana/dashboards/log-dashboard.json", + } + + ); + + local container = + engine.container("grafana") + .with_image(images.grafana) + .with_environment({ + // GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin", + // GF_AUTH_ANONYMOUS_ENABLED: "true", + // GF_ORG_ROLE: "Admin", + GF_ORG_NAME: "trustgraph.ai", + // GF_SERVER_ROOT_URL: "https://example.com", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(3000, 3000, "cassandra") + .with_volume_mount(vol, "/var/lib/grafana") + .with_volume_mount( + provDashVol, "/etc/grafana/provisioning/dashboards/" + ) + .with_volume_mount( + provDataVol, "/etc/grafana/provisioning/datasources/" + ) + .with_volume_mount( + dashVol, "/var/lib/grafana/dashboards/" + ); + + local containerSet = engine.containers( + "grafana", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3000, 3000, "http"); + + engine.resources([ + vol, + provDashVol, + provDataVol, + dashVol, + containerSet, + service, + ]) + + }, + +} + loki + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/monitoring/loki.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/monitoring/loki.jsonnet new file mode 100644 index 00000000..0774f6cf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/monitoring/loki.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "loki" +: { + + create:: function(engine) + + local vol = engine.volume("loki-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "loki-cfg", "loki", + { + "local-config.yaml": importstr "loki/local-config.yaml", + } + ); + + local container = + engine.container("loki") + .with_image(images.loki) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(3100, 3100, "http") + .with_volume_mount(cfgVol, "/etc/loki/") + .with_volume_mount(vol, "/loki"); + + local containerSet = engine.containers( + "loki", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3100, 3100, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/ocr/mistral-ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/ocr/mistral-ocr.jsonnet new file mode 100644 index 00000000..f70e0cd6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/ocr/mistral-ocr.jsonnet @@ -0,0 +1,49 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "pdf-decoder" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("mistral-ocr") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-ocr-mistral", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "mistral-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/ocr/ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/ocr/ocr.jsonnet new file mode 100644 index 00000000..4f53aa5a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/ocr/ocr.jsonnet @@ -0,0 +1,37 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "pdf-decoder" +: { + + create:: function(engine) + + local container = + engine.container("pdf-ocr") + .with_image(images.trustgraph_ocr) + .with_command([ + "pdf-ocr", + "-p", + url.pulsar, + ]) + .with_limits("1.0", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "pdf-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/azure-openai.jsonnet new file mode 100644 index 00000000..0e3ea907 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/azure-openai.jsonnet @@ -0,0 +1,9 @@ +// Azure OpenAI LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-4o", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/azure.jsonnet new file mode 100644 index 00000000..6ffa5fdf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/azure.jsonnet @@ -0,0 +1,9 @@ +// Azure LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/bedrock.jsonnet new file mode 100644 index 00000000..df930e59 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/bedrock.jsonnet @@ -0,0 +1,63 @@ +// AWS Bedrock LLM Model Definitions +// Defines available models and their configurations for AWS Bedrock + +{ + "type": "string", + "description": "LLM model to use", + "default": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + "enum": [ + { + id: "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + description: "Claude Sonnet 4.5 (smartest for complex agents and coding)" + }, + { + id: "global.anthropic.claude-opus-4-5-20251101-v1:0", + description: "Claude Opus 4.5 (maximum intelligence)" + }, + { + id: "global.anthropic.claude-haiku-4-5-20251001-v1:0", + description: "Claude Haiku 4.5 (fastest with near-frontier intelligence)" + }, + { + id: "global.anthropic.claude-opus-4-1-20250805-v1:0", + description: "Claude Opus 4.1 (specialized reasoning)" + }, + { + id: "global.anthropic.claude-sonnet-4-20250514-v1:0", + description: "Claude Sonnet 4.0" + }, + { + id: "global.anthropic.claude-opus-4-20250514-v1:0", + description: "Claude Opus 4.0" + }, + { + id: "anthropic.claude-3-5-haiku-20241022-v1:0", + description: "Claude 3.5 Haiku" + }, + { + id: "anthropic.claude-3-haiku-20240307-v1:0", + description: "Claude 3 Haiku" + }, + { + id: "meta.llama3-1-405b-instruct-v1:0", + description: "Llama 3.1 405B Instruct" + }, + { + id: "meta.llama3-1-70b-instruct-v1:0", + description: "Llama 3.1 70B Instruct" + }, + { + id: "meta.llama3-1-8b-instruct-v1:0", + description: "Llama 3.1 8B Instruct" + }, + { + id: "mistral.mistral-large-2407-v1:0", + description: "Mistral Large" + }, + { + id: "mistral.mixtral-8x7b-instruct-v0:1", + description: "Mixtral 8x7B Instruct" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/chunking-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/chunking-param-types.jsonnet new file mode 100644 index 00000000..f82fa3f7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/chunking-param-types.jsonnet @@ -0,0 +1,25 @@ +// Chunk parameter type definitions + +{ + "chunk-size": { + "type": "integer", + "description": "Chunk size", + "placeholder": 2000, + "helper": "An integer, usually 2000 .. 8000", + "default": 2000, + "min": 0, + "max": 32768, + "required": true + }, + "chunk-overlap": { + "type": "integer", + "description": "Chunk overlap", + "placeholder": 50, + "helper": "An integer, usually 50 .. 100", + "default": 50, + "min": 0, + "max": 8000, + "required": true + }, +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/claude.jsonnet new file mode 100644 index 00000000..40fd434b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/claude.jsonnet @@ -0,0 +1,35 @@ +// Claude LLM Model Definitions +// Defines available models and their configurations for Anthropic's Claude + +{ + "type": "string", + "description": "LLM model to use", + "default": "claude-sonnet-4-5-20250929", + "enum": [ + { + id: "claude-sonnet-4-5-20250929", + description: "Claude Sonnet 4.5 (complex agents + coding)" + }, + { + id: "claude-opus-4-5-20251101", + description: "Claude Opus 4.5 (maximum intelligence)" + }, + { + id: "claude-haiku-4-5-20251001", + description: "Claude Haiku 4.5 (fast)" + }, + { + id: "claude-opus-4-1-20250805", + description: "Claude Opus 4.1 (specialized reasoning)" + }, + { + id: "claude-sonnet-4-20250514", + description: "Claude Sonnet 4.0" + }, + { + id: "claude-3-5-haiku-20241022", + description: "Claude 3.5 Haiku" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/cohere.jsonnet new file mode 100644 index 00000000..5a5865a3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/cohere.jsonnet @@ -0,0 +1,35 @@ +// Cohere LLM Model Definitions +// Defines available models and their configurations for Cohere + +{ + "type": "string", + "description": "LLM model to use", + "default": "command-r-plus-08-2024", + "enum": [ + { + id: "command-r-plus-08-2024", + description: "Command R+ (August 2024)" + }, + { + id: "command-r-08-2024", + description: "Command R (August 2024)" + }, + { + id: "command-r-plus", + description: "Command R+ (legacy)" + }, + { + id: "command-r", + description: "Command R (legacy)" + }, + { + id: "command", + description: "Command" + }, + { + id: "command-light", + description: "Command Light" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..477e25dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/embeddings-fastembed.jsonnet @@ -0,0 +1,127 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "sentence-transformers/all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "BAAI/bge-small-en-v1.5", + "description": "bge-small-en-v1.5" + }, + { + "id": "BAAI/bge-small-zh-v1.5", + "description": "bge-small-zh-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-xs", + "description": "snowflake-arctic-embed-xs" + }, + { + "id": "jinaai/jina-embeddings-v2-small-en", + "description": "jina-embeddings-v2-small-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "snowflake/snowflake-arctic-embed-s", + "description": "snowflake-arctic-embed-s" + }, + { + "id": "BAAI/bge-small-en", + "description": "bge-small-en" + }, + { + "id": "BAAI/bge-base-en-v1.5", + "description": "bge-base-en-v1.5" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "description": "paraphrase-multilingual-MiniLM-L12-v2" + }, + { + "id": "Qdrant/clip-ViT-B-32-text", + "description": "clip-ViT-B-32-text" + }, + { + "id": "jinaai/jina-embeddings-v2-base-de", + "description": "jina-embeddings-v2-base-de" + }, + { + "id": "BAAI/bge-base-en", + "description": "bge-base-en" + }, + { + "id": "snowflake/snowflake-arctic-embed-m", + "description": "snowflake-arctic-embed-m" + }, + { + "id": "thenlper/gte-base", + "description": "gte-base" + }, + { + "id": "jinaai/jina-embeddings-v2-base-en", + "description": "jina-embeddings-v2-base-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1", + "description": "nomic-embed-text-v1" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5", + "description": "nomic-embed-text-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-m-long", + "description": "snowflake-arctic-embed-m-long" + }, + { + "id": "jinaai/jina-clip-v1", + "description": "jina-clip-v1" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + { + "id": "jinaai/jina-embeddings-v2-base-es", + "description": "jina-embeddings-v2-base-es" + }, + { + "id": "jinaai/jina-embeddings-v2-base-code", + "description": "jina-embeddings-v2-base-code" + }, + { + "id": "jinaai/jina-embeddings-v2-base-zh", + "description": "jina-embeddings-v2-base-zh" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "description": "paraphrase-multilingual-mpnet-base-v2" + }, + { + "id": "snowflake/snowflake-arctic-embed-l", + "description": "snowflake-arctic-embed-l" + }, + { + "id": "BAAI/bge-large-en-v1.5", + "description": "bge-large-en-v1.5" + }, + { + "id": "thenlper/gte-large", + "description": "gte-large" + }, + { + "id": "intfloat/multilingual-e5-large", + "description": "multilingual-e5-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/embeddings-huggingface.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/embeddings-huggingface.jsonnet new file mode 100644 index 00000000..32254a2b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/embeddings-huggingface.jsonnet @@ -0,0 +1,31 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "all-mpnet-base-v2", + "description": "all-mpnet-base-v2" + }, + { + "id": "all-distilroberta-v1", + "description": "all-distilroberta-v1" + }, + { + "id": "stsb-bert-large", + "description": "stsb-bert-large" + }, + { + "id": "sentence-camembert-large", + "description": "sentence-camembert-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/embeddings-ollama.jsonnet new file mode 100644 index 00000000..c3f29fbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/embeddings-ollama.jsonnet @@ -0,0 +1,23 @@ +// Embeddings model definitions for Ollama +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "all-minilm:latest", + "enum": [ + { + "id": "all-minilm:latest", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/googleaistudio.jsonnet new file mode 100644 index 00000000..a8891932 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/googleaistudio.jsonnet @@ -0,0 +1,67 @@ +// Google AI Studio LLM Model Definitions +// Defines available models and their configurations for Google AI Studio + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 2.5 models (latest generation) + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-exp", + description: "Gemini 2.0 Flash (experimental)" + }, + + + // Claude models on VertexAI + { + id: "claude-3-5-sonnet@20241022", + description: "Claude 3.5 Sonnet (via VertexAI)" + }, + { + id: "claude-3-5-haiku@20241022", + description: "Claude 3.5 Haiku (via VertexAI)" + }, + { + id: "claude-3-opus@20240229", + description: "Claude 3 Opus (via VertexAI)" + }, + { + id: "claude-3-sonnet@20240229", + description: "Claude 3 Sonnet (via VertexAI)" + }, + { + id: "claude-3-haiku@20240307", + description: "Claude 3 Haiku (via VertexAI)" + }, + + // Llama models on VertexAI + { + id: "llama3-405b-instruct-maas", + description: "Llama 3 405B Instruct (via VertexAI)" + }, + { + id: "llama3-70b-instruct-maas", + description: "Llama 3 70B Instruct (via VertexAI)" + }, + { + id: "llama3-8b-instruct-maas", + description: "Llama 3 8B Instruct (via VertexAI)" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/llamafile.jsonnet new file mode 100644 index 00000000..f6480aa9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/llamafile.jsonnet @@ -0,0 +1,9 @@ +// LlamaFile LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/lmstudio.jsonnet new file mode 100644 index 00000000..1c88da3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/lmstudio.jsonnet @@ -0,0 +1,150 @@ +// LMStudio LLM Model Definitions +// Defines available models and their configurations for LMStudio + +{ + "type": "string", + "description": "LLM model to use", + "default": "llama3.1:70b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/mistral.jsonnet new file mode 100644 index 00000000..07f75b81 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/mistral.jsonnet @@ -0,0 +1,61 @@ +// Mistral LLM Model Definitions +// Defines available models and their configurations for Mistral AI + +{ + "type": "string", + "description": "LLM model to use", + "default": "mistral-medium-2508", + "enum": [ + // Featured models + { + id: "mistral-medium-2508", + description: "Mistral Medium 3.1" + }, + { + id: "mistral-large-2512", + description: "Mistral Large 3" + }, + { + id: "mistral-small-2506", + description: "Mistral Small 3.2" + }, + { + id: "ministral-14b-2512", + description: "Ministral 3 14B" + }, + { + id: "ministral-8b-2512", + description: "Ministral 3 8B" + }, + { + id: "ministral-3b-2512", + description: "Ministral 3 3B" + }, + { + id: "magistral-medium-2509", + description: "Magistral Medium 1.2 (reasoning)" + }, + { + id: "magistral-small-2509", + description: "Magistral Small 1.2 (reasoning)" + }, + { + id: "devstral-2512", + description: "Devstral 2 (code)" + }, + // Other models + { + id: "codestral-2508", + description: "Codestral (code)" + }, + { + id: "pixtral-large-2411", + description: "Pixtral Large (vision)" + }, + { + id: "open-mistral-nemo", + description: "Open Mistral Nemo" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/ollama.jsonnet new file mode 100644 index 00000000..4ef98a6a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/ollama.jsonnet @@ -0,0 +1,150 @@ +// Ollama LLM Model Definitions +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemma3:12b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/openai.jsonnet new file mode 100644 index 00000000..75b2670f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/openai.jsonnet @@ -0,0 +1,31 @@ +// OpenAI LLM Model Definitions +// Defines available models and their configurations for OpenAI's platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-4o", + "enum": [ + { + id: "gpt-4o", + description: "GPT-4o (latest)" + }, + { + id: "gpt-4o-mini", + description: "GPT-4o Mini" + }, + { + id: "gpt-4-turbo", + description: "GPT-4 Turbo" + }, + { + id: "gpt-4", + description: "GPT-4" + }, + { + id: "gpt-3.5-turbo", + description: "GPT-3.5 Turbo" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/temperature-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/temperature-param-types.jsonnet new file mode 100644 index 00000000..f5523e88 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/temperature-param-types.jsonnet @@ -0,0 +1,15 @@ +// LLM temperature definitions + +{ + "llm-temperature": { + "type": "float", + "description": "LLM temperature", + "placeholder": 0.3, + "helper": "A floating point number between 0 and 1", + "default": 0.3, + "min": 0.0, + "max": 10.0, + "required": true + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/vertexai.jsonnet new file mode 100644 index 00000000..6d90f8d3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/vertexai.jsonnet @@ -0,0 +1,68 @@ +// VertexAI LLM Model Definitions +// Defines available models and their configurations for Google's VertexAI platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 2.5 models (latest generation) + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-exp", + description: "Gemini 2.0 Flash (experimental)" + }, + + + // Claude models on VertexAI + { + id: "claude-3-5-sonnet@20241022", + description: "Claude 3.5 Sonnet (via VertexAI)" + }, + { + id: "claude-3-5-haiku@20241022", + description: "Claude 3.5 Haiku (via VertexAI)" + }, + { + id: "claude-3-opus@20240229", + description: "Claude 3 Opus (via VertexAI)" + }, + { + id: "claude-3-sonnet@20240229", + description: "Claude 3 Sonnet (via VertexAI)" + }, + { + id: "claude-3-haiku@20240307", + description: "Claude 3 Haiku (via VertexAI)" + }, + + // Llama models on VertexAI + { + id: "llama3-405b-instruct-maas", + description: "Llama 3 405B Instruct (via VertexAI)" + }, + { + id: "llama3-70b-instruct-maas", + description: "Llama 3 70B Instruct (via VertexAI)" + }, + { + id: "llama3-8b-instruct-maas", + description: "Llama 3 8B Instruct (via VertexAI)" + }, + ], + "required": true +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/vllm.jsonnet new file mode 100644 index 00000000..120342b2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/parameters/vllm.jsonnet @@ -0,0 +1,18 @@ +// vLLM Model Definitions +// Defines available models and their configurations for vLLM +// vLLM works with any HuggingFace model. We have to use the model +// vLLM was initialised with + +{ + "type": "string", + "description": "LLM model to use", + "default": "model", + "enum": [ + // Llama 3.1 models + { + id: "model", + description: "Pre-defined model" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/profiles/memory-profile-low.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/profiles/memory-profile-low.jsonnet new file mode 100644 index 00000000..e6341062 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/profiles/memory-profile-low.jsonnet @@ -0,0 +1,169 @@ +// Low memory profile - reduces memory allocation across components. +// Include this at the END of your configuration to override default memory +// settings with lower values suitable for memory-constrained environments. +// +// Usage: {"name": "memory-profile-low", "parameters": {}} +// +// Note: This trades some performance headroom for reduced memory usage. +// Monitor for OOM errors under heavy load. + +{ + + // Override Pulsar stack memory settings + "pulsar" +: { + + // Zookeeper: 512M -> 300M + "zk-memory-limit":: "300M", + "zk-memory-reservation":: "200M", + "zk-heap":: "128m", + "zk-direct-memory":: "64m", + + // Bookie: 1024M -> 600M + "bookie-memory-limit":: "600M", + "bookie-memory-reservation":: "400M", + "bookie-heap":: "128m", + "bookie-direct-memory":: "128m", + + // Broker: 800M -> 512M + "broker-memory-limit":: "512M", + "broker-memory-reservation":: "400M", + "broker-heap":: "192m", + "broker-direct-memory":: "192m", + + // Pulsar-init: 256M -> 128M + "init-memory-limit":: "128M", + "init-memory-reservation":: "128M", + "init-heap":: "64m", + "init-direct-memory":: "64m", + + }, + + // Override Cassandra memory settings: 1000M -> 600M + "cassandra" +: { + "memory-limit":: "600M", + "memory-reservation":: "500M", + "heap":: "200M", + }, + + // Override Qdrant memory settings: 1024M -> 600M + // Also enables mmap for vectors/payloads (trades latency for memory) + "qdrant" +: { + "memory-limit":: "600M", + "memory-reservation":: "500M", + "memmap-threshold-kb":: "1", + "on-disk-payload":: "true", + }, + + // TrustGraph core services - 50% memory reservations + "api-gateway" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "chunker" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "config-svc" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "pdf-decoder" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "mcp-tool" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "mcp-server" +: { + "memory-reservation":: "128M", // 256M -> 128M + }, + + "metering" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "metering-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-store" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-manager" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "prompt" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "prompt-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-embeddings" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "librarian" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "agent-manager" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + // Graph RAG services + "kg-extract-definitions" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-relationships" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-agent" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-ontology" +: { + "memory-reservation":: "150M", // 300M -> 150M + }, + + "kg-extract-objects" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "graph-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "graph-embeddings" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + // Structured data services + "nlp-query" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "structured-query" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "structured-diag" +: { + "memory-reservation":: "48M", // 96M -> 48M + }, + + // Init service + "init-trustgraph" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/agent-kg-extract.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/agent-kg-extract.txt new file mode 100644 index 00000000..36dbd74f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/agent-kg-extract.txt @@ -0,0 +1,28 @@ +Analyze the following text and extract both entity definitions and relationships. + +For definitions, extract entities and their explanations or descriptions. +For relationships, extract subject-predicate-object triples where subjects and objects are entities, and predicates are relationship types. + +Text: {{text}} + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include a "type" field to distinguish between definitions and relationships. + +For definitions, output: +{"type": "definition", "entity": "entity_name", "definition": "definition_text"} + +For relationships, output: +{"type": "relationship", "subject": "subject_entity", "predicate": "relationship_type", "object": "object_entity_or_literal", "object-entity": true} + +Requirements: +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations, only output JSONL + +Example output: +{"type": "definition", "entity": "DNA", "definition": "Deoxyribonucleic acid, a molecule carrying genetic instructions"} +{"type": "definition", "entity": "RNA", "definition": "Ribonucleic acid, essential for coding and gene expression"} +{"type": "relationship", "subject": "DNA", "predicate": "transcribes_to", "object": "RNA", "object-entity": true} +{"type": "relationship", "subject": "DNA", "predicate": "located_in", "object": "cell nucleus", "object-entity": true} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/agent-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/agent-prompt.txt new file mode 100644 index 00000000..9155d20f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/agent-prompt.txt @@ -0,0 +1,104 @@ +# ReAct Agent System Prompt + +You are an AI assistant that uses the ReAct (Reasoning + Acting) framework to solve problems through systematic reasoning and tool use. + +## Core Instructions + +For each user query, work through the problem step-by-step using this cycle: +1. **Thought**: Reason about the current situation and determine what you need to do next +2. **Action**: Take ONE specific action using an available tool +3. Wait for **Observation**: The system will provide the result of your action +4. Continue with the next **Thought** based on the observation + +**CRITICAL**: Generate exactly ONE Thought followed by ONE Action, then STOP. Do not generate multiple Thought/Action pairs in a single response. Do not generate Observations yourself - the system will provide them. + +## Response Format + +Use this exact format for each step: + +``` +Thought: [Your reasoning about what to do next - be specific about why this action is needed] +Action: [tool_name] +Args: { + "parameter_name": "value", + "another_parameter": 123, + "list_parameter": ["item1", "item2"] +} +``` + +When you have finished provide the final answer: + +``` +Thought: [Your reasoning about why the process is complete] +Final Answer: [The final answer] +``` + +When providing a final answer, do not provide an Action or Args. + +## Action Format Rules + +1. **Tool Name**: Write "Action: " followed by the exact tool name on its own line +2. **Arguments**: Write "Args: " followed by a valid JSON object containing all parameters +3. **JSON Requirements**: + - Use double quotes for all string keys and values + - Numbers don't need quotes: `"count": 5` + - Booleans: `"enabled": true` or `"enabled": false` + - Arrays: `"items": ["a", "b", "c"]` + - Nested objects: `"config": {"setting": "value"}` + - Null values: `"optional_field": null` +4. **Required Parameters**: Include all required parameters for the tool +5. **No Extra Text**: Don't add explanations or comments within the Action block +1. **Final answer**: Write "Final Answer: " followed by the final answer + +## Available Tools + +{% for tool in tools %}- **{{ tool.name }}**: {{ tool.description }} +{% for arg in tool.arguments %} - Required: `"{{ arg.name }}"` ({{ arg.type }}): {{ arg.description }} +{% endfor %} +{% endfor %} + +## Behavior Rules + +1. **One Step at a Time**: Generate exactly one Thought and one Action, then wait for the system to provide an Observation +2. **Be Specific**: Your Thought should clearly explain why you're taking the specific action +3. **Use Context**: Build on previous Observations to inform your next steps +4. **Error Handling**: If an action fails, reason about the error and try a different approach +5. **Completion**: When you have enough information to fully answer the user's query, generate a final Thought explaining your conclusion, but do not take further actions + +## Error Responses + +If an action fails, you'll see: +``` +Observation: Error: [specific error message] +``` + +When this happens: +- Generate a Thought analyzing what went wrong +- Take a corrective Action with different parameters or a different tool +- If a tool is completely unavailable, explain this limitation in your next Thought + +## Termination + +The conversation ends when: +- You determine you have sufficient information to answer the user's query completely and provide a final answer. +- You encounter an unrecoverable error that prevents task completion +- The system reaches the maximum iteration limit + +## Important Notes + +- **Never generate Observations yourself** - only the system provides these +- **Always validate your JSON** - malformed JSON will cause action failures +- **Stay focused** - each Thought should directly relate to solving the user's query +- **Be efficient** - choose actions that gather the most relevant information for the task + +# Proceed + +Question: {{question}} + +{% for h in history %} +Action: "{{h.action}}" +Args: { +{% for k, v in h.arguments.items() %} "{{k}}": "{{v}}" +{% endfor %}} +Observation: "{{h.observation}}" +{% endfor %} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/cohere.jsonnet new file mode 100644 index 00000000..9541e4c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/cohere.jsonnet @@ -0,0 +1,42 @@ +// For Cohere. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/default-prompts.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/default-prompts.jsonnet new file mode 100644 index 00000000..3b173a5a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/default-prompts.jsonnet @@ -0,0 +1,233 @@ + +// Prompt templates. For tidy JSONNET use, don't change these templates +// here, but use over-rides in the prompt directory + +{ + + "system-template":: "You are a helpful assistant.", + + "templates":: { + + "question":: { + "prompt": "{{question}}", + }, + + "extract-definitions":: { + "prompt": importstr "extract-definitions.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "entity": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "entity", + "definition" + ] + } + }, + + "extract-relationships":: { + "prompt": importstr "extract-relationships.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "object": { + "type": "string" + }, + "object-entity": { + "type": "boolean" + } + }, + "required": [ + "subject", + "predicate", + "object", + "object-entity" + ] + } + }, + + "extract-topics":: { + "prompt": importstr "extract-topics.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "topic": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "topic", + "definition" + ] + } + }, + + "extract-rows":: { + "prompt": importstr "extract-rows.txt", + "response-type": "jsonl", + }, + + "kg-prompt":: { + "prompt": importstr "kg-prompt.txt", + "response-type": "text", + }, + + "document-prompt":: { + "prompt": importstr "document-prompt.txt", + "response-type": "text", + }, + + "agent-react":: { + "prompt": importstr "agent-prompt.txt", + "response-type": "text" + }, + + "agent-kg-extract":: { + "prompt": importstr "agent-kg-extract.txt", + "response-type": "jsonl", + "object-schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { "const": "definition" }, + "entity": { "type": "string" }, + "definition": { "type": "string" } + }, + "required": ["type", "entity", "definition"] + }, + { + "type": "object", + "properties": { + "type": { "const": "relationship" }, + "subject": { "type": "string" }, + "predicate": { "type": "string" }, + "object": { "type": "string" }, + "object-entity": { "type": "boolean" } + }, + "required": ["type", "subject", "predicate", "object"] + } + ] + } + }, + + "schema-selection":: { + "prompt": importstr "schema-selection.txt", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "An array of schema names that are relevant to answering the given question" + } + }, + + "graphql-generation":: { + "prompt": importstr "graphql-generation.txt", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The GraphQL query string generated to answer the question" + }, + "variables": { + "type": "object", + "description": "Object containing any GraphQL variables needed for the query", + "additionalProperties": true + }, + "confidence": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "description": "Float between 0.0-1.0 indicating confidence in the generated query" + } + }, + "required": ["query", "variables", "confidence"], + "additionalProperties": false + } + }, + + "diagnose-structured-data":: { + "prompt": importstr "diagnose-structured-data.txt", + "response-type": "json", + }, + + "diagnose-xml":: { + "prompt": importstr "diagnose-xml.txt", + "response-type": "json", + }, + "diagnose-json":: { + "prompt": importstr "diagnose-json.txt", + "response-type": "json", + }, + "diagnose-csv":: { + "prompt": importstr "diagnose-csv.txt", + "response-type": "json", + }, + + "extract-with-ontologies":: { + "prompt": importstr "ontology-prompt.txt", + "response-type": "jsonl", + "object-schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { "const": "entity" }, + "entity": { "type": "string" }, + "entity_type": { "type": "string" } + }, + "required": ["type", "entity", "entity_type"] + }, + { + "type": "object", + "properties": { + "type": { "const": "relationship" }, + "subject": { "type": "string" }, + "subject_type": { "type": "string" }, + "relation": { "type": "string" }, + "object": { "type": "string" }, + "object_type": { "type": "string" } + }, + "required": ["type", "subject", "subject_type", "relation", "object", "object_type"] + }, + { + "type": "object", + "properties": { + "type": { "const": "attribute" }, + "entity": { "type": "string" }, + "entity_type": { "type": "string" }, + "attribute": { "type": "string" }, + "value": { "type": "string" } + }, + "required": ["type", "entity", "entity_type", "attribute", "value"] + } + ] + } + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-csv.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-csv.txt new file mode 100644 index 00000000..ad9725ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-csv.txt @@ -0,0 +1,370 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in CSV processing and delimiter-separated value formats. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured CSV data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## CSV Processing Expertise + +When working with CSV data, you must: + +1. **Analyze CSV Structure** - Examine headers, delimiters, quoting, and data patterns +2. **Identify Column Mappings** - Map source column names to target fields +3. **Handle Complex CSV Patterns** - Support various CSV formats including: + - Standard comma-separated values: `name,age,city` + - Alternative delimiters: tab-separated (TSV), pipe-separated, semicolon-separated + - Quoted fields with embedded delimiters: `"Last, First",25,"New York, NY"` + - Headers with spaces or special characters: `"Customer Name","Order Date","Total Amount"` + - Files with or without headers + - Multi-line fields with embedded newlines + +## CSV Format Configuration Guidelines + +For CSV format configurations, use these patterns: + +**Basic CSV Configuration:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "skip_rows": 0 + } + } +} +``` + +**Advanced CSV Options:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": "\t", // Tab-separated + "quote_char": "\"", + "escape_char": "\\", + "has_header": true, + "skip_rows": 2, // Skip metadata rows + "null_values": ["", "NULL", "N/A"], + "trim_whitespace": true, + "skip_blank_lines": true + } + } +} +``` + +**CRITICAL: Source Field Names in Mappings** + +When processing CSV files, the parser uses column headers (if present) or generates column indices as field names. Your source field names in mappings must match exactly: + +**CORRECT Example with Headers:** +CSV file: +```csv +Customer Name,Order Date,Total Amount,Status +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "Customer Name": "John Smith", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Customer Name", // βœ… Correct - matches header exactly + "source_field": "Order Date", // βœ… Correct - matches header exactly + "source_field": "Total Amount", // βœ… Correct - matches header exactly + "source_field": "Status" // βœ… Correct - matches header exactly +} +``` + +**CORRECT Example without Headers:** +CSV file without headers uses column indices: +```csv +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "0": "John Smith", + "1": "2024-01-15", + "2": "1000.50", + "3": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "0", // βœ… Correct - first column + "source_field": "1", // βœ… Correct - second column + "source_field": "2", // βœ… Correct - third column + "source_field": "3" // βœ… Correct - fourth column +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - Delimiter character (comma, tab, pipe, semicolon, etc.) + - Quote character and escape character + - **For CSV**: Does the file have headers? Sample structure + - Text encoding (UTF-8, Windows-1252, etc.) + - Any rows to skip (metadata, blank lines) + - How null/empty values are represented + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source column β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Date/time format conversions + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + - Row-level validation rules + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## CSV Structure Analysis + +When presented with CSV data, analyze: + +1. **Delimiter Detection**: What character separates the fields? +2. **Header Presence**: Does the first row contain column names? +3. **Quote Pattern**: Are fields quoted? What quote character is used? +4. **Data Types**: What types are present in each column? +5. **Null Representation**: How are empty/null values represented? +6. **Special Characters**: Are there embedded commas, quotes, or newlines? +7. **Encoding Issues**: Are there any character encoding problems? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + // CSV-specific parsing options + // delimiter, quote_char, has_header, skip_rows, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `split`, `strip_quotes` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_number`, `parse_currency` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `clean_whitespace`, `normalize_encoding` + +**Date/Time Operations:** +- `parse_date`, `format_date`, `date_component` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `numeric_range`, `date_range` + +## CSV-Specific Best Practices + +1. **Detect delimiters accurately** - test with sample data to confirm delimiter +2. **Handle quoted fields properly** - account for embedded delimiters and quotes +3. **Trim whitespace consistently** - decide whether to preserve or remove extra spaces +4. **Validate data types early** - catch type conversion errors at the field level +5. **Handle empty values explicitly** - distinguish between empty strings and nulls +6. **Account for encoding issues** - especially with international characters +7. **Validate row structure** - ensure consistent column counts across rows + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle CSV-specific edge cases** like malformed quotes, inconsistent delimiters +10. **Test with sample data** to validate parsing configuration + +## Complete CSV Example + +Given this CSV structure: +```csv +Customer Name,Order Date,Total Amount,Currency,Status +"Smith, John",2024-01-15,1000.50,USD,Active +"Doe, Jane",2024-01-16,750.25,EUR,Pending +"Johnson, Bob",2024-01-17,,USD,Cancelled +``` + +The parser will: +1. Use `delimiter: ","` and `quote_char: "\""` to parse fields correctly +2. Use `has_header: true` to treat first row as column names +3. Create this parsed data structure for each record: + ```json + { + "Customer Name": "Smith, John", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Currency": "USD", + "Status": "Active" + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "trim_whitespace": true, + "null_values": ["", "NULL"] + } + }, + "mappings": [ + { + "source_field": "Customer Name", // βœ… Matches header exactly + "target_field": "customer_name", + "transforms": [{"type": "trim"}] + }, + { + "source_field": "Order Date", // βœ… Matches header exactly + "target_field": "order_date", + "transforms": [{"type": "to_date", "format": "YYYY-MM-DD"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "Total Amount", // βœ… Matches header exactly + "target_field": "amount", + "transforms": [ + {"type": "to_float"}, + {"type": "default", "value": 0.0} + ] + }, + { + "source_field": "Currency", // βœ… Matches header exactly + "target_field": "currency_code", + "transforms": [{"type": "upper"}], + "validation": [{"type": "in_list", "values": ["USD", "EUR", "GBP"]}] + }, + { + "source_field": "Status", // βœ… Matches header exactly + "target_field": "order_status", + "transforms": [{"type": "lower"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the column headers exactly, or use column indices (0, 1, 2, etc.) for files without headers.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the CSV structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to delimiter detection, header identification, quoting patterns, and data type inference: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-json.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-json.txt new file mode 100644 index 00000000..2fe6341d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-json.txt @@ -0,0 +1,327 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in JSON processing and JSONPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured JSON data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## JSON Processing Expertise + +When working with JSON data, you must: + +1. **Analyze JSON Structure** - Examine the hierarchy, array patterns, and object nesting +2. **Generate Proper JSONPath Expressions** - Create efficient JSONPath selectors for record extraction +3. **Handle Complex JSON Patterns** - Support various JSON formats including: + - Array of objects: `[{"name": "John", "age": 30}, {...}]` + - Nested object arrays: `{"data": {"records": [{"id": 1}, {...}]}}` + - Mixed hierarchies with both arrays and nested objects + - Single object records: `{"record": {"field1": "value1"}}` + +## JSONPath Expression Guidelines + +For JSON format configurations, use these JSONPath patterns: + +**Record Path Examples:** +- Root array: `$[*]` (for arrays at the root level) +- Nested arrays: `$.data.records[*]` or `$.response.items[*]` +- Single object: `$.record` (when there's one record per file) +- Deep nesting: `$.data.results.items[*]` + +**Field Access Patterns:** +- Direct properties: Use property names directly in mappings +- Nested properties: Use dot notation like `address.street` or `contact.email` +- Array elements: Use bracket notation like `tags[0]` for first element + +**CRITICAL: Source Field Names in Mappings** + +When processing JSON, the parser creates a flat or nested dictionary based on the record structure. Your source field names in mappings must match the actual property names in the parsed records: + +**CORRECT Example:** +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "metadata": { + "source": "UN", + "year": 2024 + } +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches property name + "source_field": "Trade (USD)", // βœ… Correct - matches property name + "source_field": "metadata.source", // βœ… Correct - nested property access + "source_field": "metadata.year" // βœ… Correct - nested property access +} +``` + +**JSON Format Configuration Template:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$[*]", // JSONPath to extract records + "flatten_nested": true, // Whether to flatten nested objects + "array_handling": "expand" // How to handle arrays: expand, first, concat + } + } +} +``` + +**Alternative JSON Options:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.items[*]", // For nested array structures + "flatten_nested": false, // Keep nested structure + "null_value_handling": "skip" // skip, empty_string, or preserve + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - JSON structure type (array of objects, nested objects, single records) + - **For JSON**: Sample structure, nesting patterns, array locations + - Sample data or field descriptions + - Any format-specific details (encoding, special null handling, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Nested object flattening requirements + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or null values + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## JSON Structure Analysis + +When presented with JSON data, analyze: + +1. **Root Structure**: Is it an array, object, or nested structure? +2. **Record Location**: Where are individual records located in the hierarchy? +3. **Field Pattern**: How are field names and values structured? + - Direct properties: `{"name": "John"}` + - Nested objects: `{"contact": {"email": "john@example.com"}}` + - Arrays: `{"tags": ["red", "blue"]}` + - Mixed types: `{"data": [{"id": 1, "details": {"name": "John"}}]}` +4. **Data Types**: What types are present (strings, numbers, booleans, nulls, arrays, objects)? +5. **Hierarchy Depth**: How deeply nested are the records and fields? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + // JSON-specific parsing options + // record_path (JSONPath), flatten_nested, array_handling, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `flatten_object`, `extract_array_element`, `join_array` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## JSON-Specific Best Practices + +1. **Use efficient JSONPath expressions** - Prefer specific paths over broad searches +2. **Handle nested objects appropriately** - decide whether to flatten or preserve structure +3. **Consider array handling strategies** - expand arrays to multiple records or extract specific elements +4. **Account for null vs undefined** values in field mappings +5. **Handle mixed data types** within the same field across records +6. **Use appropriate flattening** for deeply nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle JSON-specific edge cases** like empty arrays, null objects, mixed types + +## Complete JSON Example + +Given this JSON structure: +```json +{ + "data": { + "records": [ + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata": { + "source": "World Bank", + "confidence": 0.95 + } + } + ] + } +} +``` + +The parser will: +1. Use `record_path: "$.data.records[*]"` to extract record objects from the array +2. Create this parsed data structure for each record: + ```json + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata.source": "World Bank", // If flattened + "metadata.confidence": 0.95 // If flattened + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.records[*]", + "flatten_nested": true, + "array_handling": "expand" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches property name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches property name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches property name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + }, + { + "source_field": "metadata.source", // βœ… Flattened nested field + "target_field": "data_source" + } + ] +} +``` + +**KEY RULE: source_field names must match the actual property names in the parsed JSON records, using dot notation for nested properties when flattened.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the JSON structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to JSON hierarchy, object patterns, array structures, and generate appropriate JSONPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-structured-data.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-structured-data.txt new file mode 100644 index 00000000..84c4b8be --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-structured-data.txt @@ -0,0 +1,309 @@ + +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` (will be converted to relative paths automatically) +- With namespaces: `//ns:record` or `//soap:Body/data/record` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name + +**CRITICAL: Source Field Names in Mappings** + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary. Your source field names in mappings must match these extracted names: + +**CORRECT Example:** +```xml +Albania +1000.50 +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50" +} +``` + +So your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)" // βœ… Correct - matches parsed field name +} +``` + +**INCORRECT Example:** +```json +{ + "source_field": "Field[@name='Country or Area']", // ❌ Wrong - XPath not needed here + "source_field": "field[@name='Trade (USD)']" // ❌ Wrong - XPath not needed here +} +``` + +**XML Format Configuration Template:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name" // For value pattern + } + } +} +``` + +**Alternative XML Options:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // Direct element-based records + // No field_attribute needed for standard XML + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - File type (CSV, JSON, XML, Excel, fixed-width, etc.) + - **For XML**: Sample structure, namespace prefixes, record element patterns + - Sample data or field descriptions + - Any format-specific details (delimiters, encoding, namespaces, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Record Container**: Where are individual records located? +3. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns +4. **Namespaces**: Are there any namespace prefixes? +5. **Hierarchy Depth**: How deeply nested are the records? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "[csv|json|xml|fixed-width|excel]", + "encoding": "utf-8", + "options": { + // Format-specific parsing options + // For XML: record_path (XPath), field_attribute (if applicable) + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches +2. **Handle namespace prefixes** when present +3. **Identify field attribute patterns** correctly +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes + +## Complete XML Example + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "/ROOT/data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the extracted field names, NOT the XML element structure.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, and generate appropriate XPath expressions: + +{{sample}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-xml.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-xml.txt new file mode 100644 index 00000000..a3d8efa4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/diagnose-xml.txt @@ -0,0 +1,453 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for XML data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured XML data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + - CDATA sections and text nodes + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` +- With namespaces: `//ns:record` or `//soap:Body/data/record` +- Attribute-filtered: `//record[@type='customer']` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name like `field_attribute: "id"` or `field_attribute: "type"` + +## CRITICAL: Source Field Names in Mappings + +The behavior depends on whether you use `field_attribute`: + +### With field_attribute (Attribute-Based Fields) + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary: + +**Example:** +```xml + + Albania + 1000.50 + Active + +``` + +Parser configuration: +```json +{ + "record_path": "//record", + "field_attribute": "name" +} +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)", // βœ… Correct - matches parsed field name + "source_field": "Status" // βœ… Correct - matches parsed field name +} +``` + +### Without field_attribute (Element-Based Fields) + +When NOT using `field_attribute`, use direct element names or XPath expressions: + +**Example:** +```xml + + Albania + 1000.50 + Active + + UN + 2024 + + +``` + +Parser configuration: +```json +{ + "record_path": "//record" + // No field_attribute specified +} +``` + +Your mappings should use: +```json +{ + "source_field": "country", // βœ… Direct element name + "source_field": "trade_amount", // βœ… Direct element name + "source_field": "metadata/source", // βœ… Nested element path + "source_field": "metadata/year" // βœ… Nested element path +} +``` + +## XML Format Configuration Templates + +**For Attribute-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name", // Extract field names from 'name' attribute + "namespace_prefixes": { // Optional: define namespaces + "ns": "http://example.com/namespace" + } + } + } +} +``` + +**For Element-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // XPath to find record elements + "preserve_namespaces": true, // Optional: keep namespace info + "ignore_attributes": false // Optional: include element attributes + } + } +} +``` + +**For Complex XML with Namespaces:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//soap:Body//data:record", + "namespace_prefixes": { + "soap": "http://schemas.xmlsoap.org/soap/envelope/", + "data": "http://example.com/data" + }, + "field_attribute": "name" + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **XML Structure Details** + - Sample XML structure or schema + - Root element and record location + - Field organization (elements vs attributes) + - Namespace declarations and prefixes + - Text encoding (UTF-8, ISO-8859-1, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Namespaces**: Are there namespace declarations? What prefixes are used? +3. **Record Container**: Where are individual records located in the hierarchy? +4. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns: `John` +5. **Data Location**: Are values in element text, attributes, or both? +6. **Hierarchy Depth**: How deeply nested are the records? +7. **Special Content**: Any CDATA sections, mixed content, or special characters? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + // XML-specific parsing options + // record_path (XPath), field_attribute (if applicable), namespace_prefixes + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `strip_cdata`, `decode_html_entities` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_xml_date`, `parse_iso_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `extract_attribute`, `get_text_content` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `valid_xml_name`, `namespace_check` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches like `//` when possible +2. **Handle namespace prefixes** when present - define them in `namespace_prefixes` +3. **Identify field attribute patterns** correctly - determine if using `field_attribute` +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures +7. **Handle CDATA sections** and special characters properly +8. **Preserve or normalize whitespace** as appropriate +9. **Consider XML schema validation** if XSD is available + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle XML-specific edge cases** like empty elements, mixed content, processing instructions + +## Complete XML Examples + +### Example 1: Attribute-Based Fields + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "//data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +### Example 2: Element-Based Fields + +Given this XML structure: +```xml + + + John Smith + john@example.com +
+ 123 Main St + New York +
+ + + 456 + 100.50 + + +
+
+``` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer" + } + }, + "mappings": [ + { + "source_field": "name", // βœ… Direct element name + "target_field": "customer_name" + }, + { + "source_field": "email", // βœ… Direct element name + "target_field": "email_address" + }, + { + "source_field": "address/street", // βœ… Nested element path + "target_field": "street_address" + }, + { + "source_field": "address/city", // βœ… Nested element path + "target_field": "city" + }, + { + "source_field": "@id", // βœ… Attribute reference + "target_field": "customer_id", + "transforms": [{"type": "to_int"}] + } + ] +} +``` + +**KEY RULES:** +- With `field_attribute`: source_field names must match the extracted attribute values +- Without `field_attribute`: use direct element names, nested paths, or XPath expressions +- Use `@attribute` notation for XML element attributes + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, namespace usage, and generate appropriate XPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/document-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/document-prompt.txt new file mode 100644 index 00000000..5e5d65ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/document-prompt.txt @@ -0,0 +1,7 @@ +Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + +Here is the context: +{{documents}} + +Use only the provided knowledge statements to respond to the following: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-definitions.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-definitions.txt new file mode 100644 index 00000000..410ecddf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-definitions.txt @@ -0,0 +1,28 @@ + +Study the following text and derive definitions for any discovered entities. +Do not provide definitions for entities whose definitions are incomplete +or unknown. + +Output each definition as a separate JSON object on its own line (JSONL format). +Each object must have fields: +- entity: the name of the entity +- definition: English text which defines the entity + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not use special characters in the definition text +- Do not include null or unknown definitions + +Example output format: +{"entity": "photosynthesis", "definition": "The process by which plants convert sunlight into energy"} +{"entity": "chlorophyll", "definition": "Green pigment in plants that absorbs light"} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-relationships.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-relationships.txt new file mode 100644 index 00000000..d10d8118 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-relationships.txt @@ -0,0 +1,28 @@ + +Study the following text and derive entity relationships. For each +relationship, derive the subject, predicate and object of the relationship. + +Output each relationship as a separate JSON object on its own line (JSONL format). +Each object must have fields: +- subject: the subject of the relationship +- predicate: the predicate +- object: the object of the relationship +- object-entity: false if the object is a simple data type (name, value, date). true if it is an entity. + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not use special characters in the text fields + +Example output format: +{"subject": "Earth", "predicate": "orbits", "object": "Sun", "object-entity": true} +{"subject": "Earth", "predicate": "has_diameter", "object": "12742 km", "object-entity": false} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-rows.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-rows.txt new file mode 100644 index 00000000..7cea8b7d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-rows.txt @@ -0,0 +1,27 @@ + +Study the following text and derive objects which match the schema provided. + +Output each discovered object as a separate JSON object on its own line (JSONL format). +Each object's fields must carry the name field specified in the schema. + + + +{{schema}} + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object matching the schema +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations + +Example output format (assuming schema has fields: name, age, city): +{"name": "Alice", "age": 30, "city": "London"} +{"name": "Bob", "age": 25, "city": "Paris"} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-topics.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-topics.txt new file mode 100644 index 00000000..bb4b1193 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/extract-topics.txt @@ -0,0 +1,26 @@ +You are a helpful assistant that performs information extraction tasks for a provided text. +Read the provided text. You will identify topics and their definitions. + +Reading Instructions: +- Ignore document formatting in the provided text. +- Study the provided text carefully. + +Here is the text: +{{text}} + +Response Instructions: +- Output each topic as a separate JSON object on its own line (JSONL format) +- Each object must have keys "topic" and "definition" +- Do not respond with special characters +- Return only topics that are concepts and unique to the provided text + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not write any additional text or explanations + +Example output format: +{"topic": "machine learning", "definition": "A subset of AI that enables systems to learn from data"} +{"topic": "neural network", "definition": "A computing system inspired by biological neural networks"} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/gemini.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/gemini.jsonnet new file mode 100644 index 00000000..b9a1e0c0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/gemini.jsonnet @@ -0,0 +1,42 @@ +// For VertexAI Gemini. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/graphql-generation.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/graphql-generation.txt new file mode 100644 index 00000000..58af61dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/graphql-generation.txt @@ -0,0 +1,59 @@ +You are a GraphQL query generation expert. Given a natural language question and provided database schemas, generate a valid GraphQL query embedded in valid JSON. + +## Question: +{{ question }} + +## Provided Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %}- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif %}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif %}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ field.enum_values|join(', ') }}]{% endif %} +{% endfor %} +{% endfor %} + +## GraphQL Query Rules: +1. Use the schema names as GraphQL query fields (e.g., `customers`, `orders`) +2. Apply filters using the `where` parameter with nested filter objects +3. Available filter operators per field type: + - String fields: `eq`, `contains`, `startsWith`, `endsWith`, `in`, `not`, `not_in` + - Integer/Float fields: `eq`, `gt`, `gte`, `lt`, `lte`, `in`, `not`, `not_in` +4. Use `order_by` for sorting (field name as string) +5. Use `direction` for sort direction: `ASC` or `DESC` +6. Use `limit` to restrict number of results +7. Select specific fields in the query body + +## Task Instructions: +1. Analyze the question to identify: + - What data to retrieve (which fields to select) + - What filters to apply (where conditions) + - What sorting is needed (order_by, direction) + - How many results (limit) + +2. Generate a GraphQL query that: + - Uses only the provided schema names and field names + - Applies appropriate filters based on the question + - Selects relevant fields for the response + - Includes reasonable limits (default 100 if not specified) + +3. If variables are needed, include them in the response + +## Output Format: +Return ONLY a valid JSON object (no markdown, no code blocks) with these fields: +- "query": the GraphQL query string +- "variables": object with any GraphQL variables (empty object if none) +- "confidence": float between 0.0-1.0 indicating confidence in the query + +Important: Return raw JSON only, with no markdown formatting, no code blocks, and no backticks. + +## Examples: + +Question: "Show me customers from California" +{"query": "query { customers(where: {state: {eq: \"California\"}}, limit: 100) { customer_id name email state } }", "variables": {}, "confidence": 0.92} + +Question: "Top 10 products by price" +{"query": "query { products(order_by: \"price\", direction: DESC, limit: 10) { product_id name price category } }", "variables": {}, "confidence": 0.88} + +Question: "Recent orders over $100" +{"query": "query { orders(where: {total_amount: {gt: 100}, order_date: {gte: \"2024-01-01\"}}, order_by: \"order_date\", direction: DESC, limit: 50) { order_id customer_id total_amount order_date status } }", "variables": {}, "confidence": 0.96} + +Now generate the GraphQL query for the question above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/kg-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/kg-prompt.txt new file mode 100644 index 00000000..431c0d73 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/kg-prompt.txt @@ -0,0 +1,8 @@ +Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + +Here's the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}}) +{%endfor%} + +Use only the provided knowledge statements to respond to the following: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/mixtral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/mixtral.jsonnet new file mode 100644 index 00000000..cd56e7ef --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/mixtral.jsonnet @@ -0,0 +1,42 @@ +// For Mixtral. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/ontology-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/ontology-prompt.txt new file mode 100644 index 00000000..1b451d9c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/ontology-prompt.txt @@ -0,0 +1,82 @@ +You are a knowledge extraction expert. Your task is to find entities, relationships, and attributes in text based on a provided schema. + +## Entity Types + +These are the types of entities you should look for: + +{% for class_id, class_def in classes.items() %} +- **{{class_id}}**{% if class_def.subclass_of %} (subclass of {{class_def.subclass_of}}){% endif %}{% if class_def.comment %}: {{class_def.comment}}{% endif %} +{% endfor %} + +## Relationships + +These relationships connect entities to other entities: + +{% for prop_id, prop_def in object_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Attributes + +These attributes describe entity properties (text, numbers, etc.): + +{% for prop_id, prop_def in datatype_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Text to Analyze + +{{text}} + +## Your Task + +Extract the following from the text above: + +1. **Entities**: Things mentioned in the text and their types +2. **Relationships**: How entities relate to each other +3. **Attributes**: Properties of entities (like quantities, descriptions, etc.) + +## Output Format + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include a "type" field to distinguish between entities, relationships, and attributes. + +For entities, output: +{"type": "entity", "entity": "entity name as it appears in text", "entity_type": "EntityType"} + +For relationships, output: +{"type": "relationship", "subject": "subject entity name", "subject_type": "SubjectType", "relation": "relationship_name", "object": "object entity name", "object_type": "ObjectType"} + +For attributes, output: +{"type": "attribute", "entity": "entity name", "entity_type": "EntityType", "attribute": "attribute_name", "value": "literal value"} + +## Important Rules + +1. **Entity names**: Use the exact text as it appears (e.g., "Cornish pasty", "beef") +2. **Types**: Use the EXACT type identifiers from the schema above (e.g., "fo/Recipe", "fo/Food") +3. **Relationships**: Use the EXACT relationship names from the schema (e.g., "fo/has_ingredient") +4. **Attributes**: Use the EXACT attribute names from the schema (e.g., "fo/serves") +5. **No array brackets**: Output one JSON object per line, not an array +6. **No markdown**: Do not wrap output in code blocks + +## Requirements + +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting, code blocks, or prefixes +- Do not provide explanations, only output JSONL + +## Example + +Input text: "Cornish pasty is a savory pastry filled with beef and potatoes. This recipe serves 4 people." + +Expected output: +{"type": "entity", "entity": "Cornish pasty", "entity_type": "fo/Recipe"} +{"type": "entity", "entity": "beef", "entity_type": "fo/Food"} +{"type": "entity", "entity": "potatoes", "entity_type": "fo/Food"} +{"type": "relationship", "subject": "Cornish pasty", "subject_type": "fo/Recipe", "relation": "fo/has_ingredient", "object": "beef", "object_type": "fo/Food"} +{"type": "relationship", "subject": "Cornish pasty", "subject_type": "fo/Recipe", "relation": "fo/has_ingredient", "object": "potatoes", "object_type": "fo/Food"} +{"type": "attribute", "entity": "Cornish pasty", "entity_type": "fo/Recipe", "attribute": "fo/serves", "value": "4 people"} + +Now extract entities, relationships, and attributes from the text above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/openai.jsonnet new file mode 100644 index 00000000..5d232337 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/openai.jsonnet @@ -0,0 +1,42 @@ +// For OpenAI LLMs. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/schema-selection.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/schema-selection.txt new file mode 100644 index 00000000..39b180e5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/schema-selection.txt @@ -0,0 +1,25 @@ +You are a database schema selection expert. Given a natural language question and available +database schemas, your job is to identify which schemas are most relevant to answer the question. + +## Available Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}): {{ field.description }} +{% endfor %} + +{% endfor %} + +## Question: +{{ question }} + +## Instructions: +1. Analyze the question to understand what data is being requested +2. Examine each schema to understand what data it contains +3. Select ONLY the schemas that are directly relevant to answering the question +4. Return your answer as a JSON array of schema names + +## Response Format: +Return ONLY a JSON array of schema names, nothing else. +Example: ["customers", "orders", "products"] diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/slm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/slm.jsonnet new file mode 100644 index 00000000..48eb96d0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/prompts/slm.jsonnet @@ -0,0 +1,44 @@ +// For SLM. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/pulsar/pulsar-manager.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/pulsar/pulsar-manager.jsonnet new file mode 100644 index 00000000..c1ca4515 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/pulsar/pulsar-manager.jsonnet @@ -0,0 +1,40 @@ +local images = import "values/images.jsonnet"; + +{ + + "pulsar" +: { + + create:: function(engine) + +// FIXME: Should persist something? +// local volume = engine.volume(...) + + local container = + engine.container("pulsar") + .with_image(images.pulsar_manager) + .with_environment({ + SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties", + }) + .with_limits("0.5", "1.4G") + .with_reservations("0.1", "1.4G") + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2"); + + local containerSet = engine.containers( + "pulsar", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2); + + engine.resources([ + containerSet, + service, + ]) + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/pulsar/pulsar.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/pulsar/pulsar.jsonnet new file mode 100644 index 00000000..d417322d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/pulsar/pulsar.jsonnet @@ -0,0 +1,222 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +// This is a Pulsar configuration. Non-standalone mode so we deploy +// individual components: bookkeeper, broker and zookeeper. +// +// This also deploys the TrustGraph 'admin' container which initialises +// TrustGraph-specific namespaces etc. + +{ + + "pulsar" +: { + + // Zookeeper memory settings (can be overridden by memory-profile) + "zk-memory-limit":: "512M", + "zk-memory-reservation":: "512M", + "zk-heap":: "256m", + "zk-direct-memory":: "256m", + + // Bookie memory settings (can be overridden by memory-profile) + "bookie-memory-limit":: "1024M", + "bookie-memory-reservation":: "1024M", + "bookie-heap":: "256m", + "bookie-direct-memory":: "256m", + + // Broker memory settings (can be overridden by memory-profile) + "broker-memory-limit":: "800M", + "broker-memory-reservation":: "800M", + "broker-heap":: "384m", + "broker-direct-memory":: "384m", + + // Pulsar-init memory settings (can be overridden by memory-profile) + "init-memory-limit":: "256M", + "init-memory-reservation":: "256M", + "init-heap":: "128m", + "init-direct-memory":: "128m", + + create:: function(engine) + + // Capture memory settings into locals (self refers to pulsar object here) + local zkMemLimit = self["zk-memory-limit"]; + local zkMemReserv = self["zk-memory-reservation"]; + local zkHeap = self["zk-heap"]; + local zkDirect = self["zk-direct-memory"]; + + local bookieMemLimit = self["bookie-memory-limit"]; + local bookieMemReserv = self["bookie-memory-reservation"]; + local bookieHeap = self["bookie-heap"]; + local bookieDirect = self["bookie-direct-memory"]; + + local brokerMemLimit = self["broker-memory-limit"]; + local brokerMemReserv = self["broker-memory-reservation"]; + local brokerHeap = self["broker-heap"]; + local brokerDirect = self["broker-direct-memory"]; + + local initMemLimit = self["init-memory-limit"]; + local initMemReserv = self["init-memory-reservation"]; + local initHeap = self["init-heap"]; + local initDirect = self["init-direct-memory"]; + + // Zookeeper volume + local zkVolume = engine.volume("zookeeper").with_size("1G"); + + // Zookeeper container + local zkContainer = + engine.container("zookeeper") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper" + ]) + .with_limits("1", zkMemLimit) + .with_reservations("0.05", zkMemReserv) + .with_user("0:1000") + .with_volume_mount(zkVolume, "/pulsar/data/zookeeper") + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + zkHeap, zkHeap, zkDirect, + ], + }) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Pulsar cluster init container + local initContainer = + engine.container("pulsar-init") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650", + ]) + .with_limits("1", initMemLimit) + .with_reservations("0.05", initMemReserv) + .with_environment({ + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + initHeap, initHeap, initDirect, + ], + }); + + + // Bookkeeper volume + local bookieVolume = engine.volume("bookie").with_size("20G"); + + // Bookkeeper container + local bookieContainer = + engine.container("bookie") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + // false ^ causes this to be a 'failure' exit. + ]) + .with_limits("1", bookieMemLimit) + .with_reservations("0.1", bookieMemReserv) + .with_user("0:1000") + .with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper") + .with_environment({ + "clusterName": "cluster-a", + "zkServers": "zookeeper:2181", + "bookieId": "bookie", + "metadataStoreUri": "metadata-store:zk:zookeeper:2181", + "advertisedAddress": "bookie", + "BOOKIE_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + bookieHeap, bookieHeap, bookieDirect, + ], + }) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker, stateless (uses ZK and Bookkeeper for state) + local brokerContainer = + engine.container("pulsar") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" + ]) + .with_limits("1", brokerMemLimit) + .with_reservations("0.1", brokerMemReserv) + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "zookeeperServers": "zookeeper:2181", + "clusterName": "cluster-a", + "managedLedgerDefaultEnsembleSize": "1", + "managedLedgerDefaultWriteQuorum": "1", + "managedLedgerDefaultAckQuorum": "1", + "advertisedAddress": "pulsar", + "advertisedListeners": "external:pulsar://pulsar:6650,localhost:pulsar://localhost:6650", + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + brokerHeap, brokerHeap, brokerDirect, + ], + }) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + // Container sets + local zkContainerSet = engine.containers( + "zookeeper", + [ + zkContainer, + ] + ); + + local initContainerSet = engine.containers( + "init-pulsar", + [ + initContainer, + ] + ); + + local bookieContainerSet = engine.containers( + "bookie", + [ + bookieContainer, + ] + ); + + local brokerContainerSet = engine.containers( + "pulsar", + [ + brokerContainer, + ] + ); + + // Zookeeper service + local zkService = + engine.service(zkContainerSet) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Bookkeeper service + local bookieService = + engine.service(bookieContainerSet) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker service + local brokerService = + engine.service(brokerContainerSet) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + engine.resources([ + zkVolume, + bookieVolume, + zkContainerSet, + initContainerSet, + bookieContainerSet, + brokerContainerSet, + zkService, + bookieService, + brokerService, + ]) + + } + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-additionals.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-additionals.jsonnet new file mode 100644 index 00000000..3e9f3481 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-additionals.jsonnet @@ -0,0 +1,134 @@ +local decode = import "decode-config.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Custom engine that collects configVolume parts +local engine = { + + // Collection of all configVolume parts + configVolumes:: [], + + // Implement all required engine methods as no-ops + container:: function(name) { + with_image:: function(x) self, + with_command:: function(x) self, + with_environment:: function(x) self, + with_limits:: function(c, m) self, + with_reservations:: function(c, m) self, + with_port:: function(src, dest, name) self, + with_volume_mount:: function(vol, mnt) self, + with_user:: function(x) self, + with_runtime:: function(x) self, + with_privileged:: function(x) self, + with_ipc:: function(x) self, + with_capability:: function(x) self, + with_device:: function(hdev, cdev) self, + with_env_var_secrets:: function(vars) self, + }, + + volume:: function(name) { + with_size:: function(size) self, + }, + + // The key method - collects configVolume parts + configVolume:: function(name, dir, parts) + local collector = self + { + configVolumes: super.configVolumes + [ + { + dir: dir, + parts: parts, + } + ] + }; + { + // Return a dummy volume that has the collector in it + name: name, + with_size:: function(size) collector, + // Provide a way to get back to the collector + getCollector:: function() collector, + }, + + secretVolume:: function(name, dir, parts) { + with_size:: function(size) self, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self, + }, + + containers:: function(name, containers) self, + + service:: function(containers) { + with_port:: function(src, dest, name) self, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self, + }, + + resources:: function(res) + // Fold over resources and collect any configVolume state + local collected = std.foldl( + function(state, r) + if std.objectHasAll(r, 'getCollector') then + // Merge the configVolumes from the volume's collector into our state + local volumeCollector = r.getCollector(); + state + { + configVolumes: state.configVolumes + volumeCollector.configVolumes + } + else + state, + res, + self + ); + collected, +}; + +// Execute all component create() functions with our collecting engine +// Note: create:: is a hidden field, so we must use objectHasAll not objectHas +local result = std.foldl( + function(state, p) + if std.objectHasAll(p, 'create') then + // Pattern has create directly - call it + p.create(state) + else + state, + std.objectValues(patterns), + engine +); + +// Debug: show what we collected +local debug = { + numPatterns: std.length(std.objectValues(patterns)), + numConfigVolumes: std.length(result.configVolumes), +}; + +// Transform collected data into output format +local allFiles = std.flattenArrays([ + [ + { + // Remove trailing slash from dir to avoid double slashes + path: std.join("/", [std.rstripChars(cv.dir, "/"), filename]), + content: cv.parts[filename] + } + for filename in std.objectFields(cv.parts) + ] + for cv in result.configVolumes +]); + +// Deduplicate by path - use a map to keep only unique paths +local uniqueMap = std.foldl( + function(acc, item) acc + { [item.path]: item }, + allFiles, + {} +); + +// Convert back to array +local additionals = std.objectValues(uniqueMap); + +// Output the array +additionals diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-aks-k8s.jsonnet new file mode 100644 index 00000000..bcec1cbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-aks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/aks-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-docker-compose.jsonnet new file mode 100644 index 00000000..7f0dbabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-docker-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/docker-compose.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-eks-k8s.jsonnet new file mode 100644 index 00000000..f9de59cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-eks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/eks-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-gcp-k8s.jsonnet new file mode 100644 index 00000000..50037a5c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-gcp-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/gcp-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-minikube-k8s.jsonnet new file mode 100644 index 00000000..6fa64706 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-minikube-k8s.jsonnet @@ -0,0 +1,26 @@ + +local engine = import "../engine/minikube-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +// Extract resources using the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-noop.jsonnet new file mode 100644 index 00000000..3ad735d9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-noop.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/noop.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-ovh-k8s.jsonnet new file mode 100644 index 00000000..92f61041 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-ovh-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/ovh-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-podman-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-podman-compose.jsonnet new file mode 100644 index 00000000..7f0dbabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-podman-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/docker-compose.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-scw-k8s.jsonnet new file mode 100644 index 00000000..4f6c1d7a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-scw-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/scw-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-tg-configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-tg-configuration.jsonnet new file mode 100644 index 00000000..59b4732a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/config-to-tg-configuration.jsonnet @@ -0,0 +1,15 @@ + +local engine = import "../engine/noop.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract configuration directly from patterns +patterns.configuration.configuration + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/decode-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/decode-config.jsonnet new file mode 100644 index 00000000..759513c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/renderers/decode-config.jsonnet @@ -0,0 +1,31 @@ + +local components = import "../components.jsonnet"; + +local apply = function(p, components) + + local base = { + + with:: function(k, v) self + { + [k] +:: v + }, + + with_params:: function(pars) + self + std.foldl( + function(obj, par) obj.with(par.key, par.value), + std.objectKeysValues(pars), + self + ), + + }; + + local component = base + components[p.name]; + + component.with_params(p.parameters); + +local decode = function(config) + local add = function(state, c) state + apply(c, components); + local patterns = std.foldl(add, config, {}); + patterns; + +decode + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/row-store/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/row-store/cassandra.jsonnet new file mode 100644 index 00000000..7fdf433e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/row-store/cassandra.jsonnet @@ -0,0 +1,77 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "backends/cassandra.jsonnet"; + +cassandra + { + + "store-objects" +: { + + create:: function(engine) + + local container = + engine.container("store-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "objects-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-objects" +: { + + create:: function(engine) + + local container = + engine.container("query-objects") + .with_image(images.trustgraph_flow) + .with_command([ + "objects-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-objects", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/config-composer.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/config-composer.jsonnet new file mode 100644 index 00000000..e93ff044 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/config-composer.jsonnet @@ -0,0 +1,97 @@ +// Configuration Composer Module +// Orchestrates the complete configuration building process +// Combines all components into the final TrustGraph configuration + +local flow_builder = import "flow-builder.jsonnet"; +local interface_builder = import "interface-builder.jsonnet"; + +{ + // Main function to build the complete configuration + build: function(config_spec) + // Extract configuration parameters + local flow_blueprints = config_spec.flow_blueprints; + local default_flow_blueprint = config_spec.default_flow_blueprint; + local default_flow_id = config_spec.default_flow_id; + local flow_init_parameters = config_spec.flow_init_parameters; + + // Build all processors for the default flow + local blueprint_processors = flow_builder.build_blueprint_processors( + flow_blueprints, + default_flow_blueprint, + flow_init_parameters + ); + + local flow_processors = flow_builder.build_flow_processors( + flow_blueprints, + default_flow_blueprint, + default_flow_id, + flow_init_parameters + ); + + // Combine processors into flow objects + local processor_array = blueprint_processors + flow_processors; + local flow_objects = flow_builder.build_flow_objects(processor_array); + local active_flows = flow_builder.merge_flow_objects(flow_objects); + + // Build interfaces for the default flow + local default_flow_interfaces = interface_builder.build_interfaces( + flow_blueprints, + default_flow_blueprint, + default_flow_id, + flow_init_parameters + ); + + // Return object with nested configuration (for backwards compatibility) + { + // Create function (for backwards compatibility) + create: function(engine) {}, + + // The actual configuration object + configuration: { + // Prompts configuration + prompt: { + "system": config_spec.prompts["system-template"], + "template-index": std.objectFieldsAll(config_spec.prompts.templates), + } + { + ["template." + template.key]: template.value + for template in std.objectKeysValuesAll(config_spec.prompts.templates) + }, + + // Tools configuration + tool: { + [tool.id]: tool + for tool in config_spec.tools + }, + + // MCP configuration + mcp: config_spec.mcp, + + // Flow blueprints reference + "flow-blueprint": flow_blueprints, + + // Interface descriptions + "interface-description": config_spec.interface_descriptions, + + // Flow instances + "flow": { + [default_flow_id]: { + "description": "Default processing flow", + "blueprint-name": default_flow_blueprint, + "interfaces": default_flow_interfaces, + "parameters": flow_init_parameters, + }, + }, + + // Active flow processors + "active-flow": active_flows, + + // Token costs and parameter types + "token-cost": config_spec.token_costs, + "parameter-type": config_spec.parameter_types, + + // Collections configuration + "collection": config_spec.collection, + + }, + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/flow-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/flow-builder.jsonnet new file mode 100644 index 00000000..0b142750 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/flow-builder.jsonnet @@ -0,0 +1,72 @@ +// Flow Builder Module +// Processes flow blueprints and builds complete flow configurations +// Handles {blueprint}, {id}, and parameter substitutions + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds blueprint-level processors with parameter substitution + // Processes the 'blueprint' section of flow blueprints + build_blueprint_processors: function(flow_blueprints, blueprint_name, parameters) + [ + [ + // Replace {blueprint} in the processor key + local key = std.strReplace(processor.key, "{blueprint}", blueprint_name); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // First replace {blueprint}, then substitute parameters + local blueprint_replaced = std.strReplace(field.value, "{blueprint}", blueprint_name); + param_processor.substitute_parameters(blueprint_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_blueprints[blueprint_name].blueprint) + ], + + // Builds flow-level processors with parameter substitution + // Processes the 'flow' section of flow blueprints + build_flow_processors: function(flow_blueprints, blueprint_name, flow_id, parameters) + [ + [ + // Replace both {blueprint} and {id} in the processor key + local key = std.strReplace( + std.strReplace(processor.key, "{blueprint}", blueprint_name), + "{id}", flow_id + ); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // Replace {blueprint} and {id}, then substitute parameters + local blueprint_replaced = std.strReplace(field.value, "{blueprint}", blueprint_name); + local id_replaced = std.strReplace(blueprint_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_blueprints[blueprint_name].flow) + ], + + // Combines blueprint and flow processors into flow objects + build_flow_objects: function(processor_array) + std.map( + function(item) { + [item[0][0]] +: { + [item[0][1]]: item[1] + } + }, + processor_array + ), + + // Merges all flow objects into a single flows_active configuration + merge_flow_objects: function(flow_objects) + std.foldr( + function(a, b) a + b, + flow_objects, + {} + ), +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/interface-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/interface-builder.jsonnet new file mode 100644 index 00000000..2143e600 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/interface-builder.jsonnet @@ -0,0 +1,30 @@ +// Interface Builder Module +// Processes flow class interfaces with parameter substitution +// Handles both string interfaces and nested object interfaces + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds interfaces for a specific flow class and instance + // Processes the 'interfaces' section of flow classes + build_interfaces: function(flow_classes, class_name, flow_id, parameters) + local interface_spec = flow_classes[class_name].interfaces; + { + [interface.key]: + if std.isString(interface.value) then + // Simple string interface - apply all substitutions + local class_replaced = std.strReplace(interface.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + else + // Complex object interface - process nested fields + { + [field.key]: + local class_replaced = std.strReplace(field.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(interface.value) + } + for interface in std.objectKeysValuesAll(interface_spec) + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/interface-descriptions.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/interface-descriptions.jsonnet new file mode 100644 index 00000000..f9dda8b0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/interface-descriptions.jsonnet @@ -0,0 +1,89 @@ +// Interface Descriptions Module +// Defines all external interfaces available in TrustGraph flows +// These are the 'endpoints' that external systems can interact with + +{ + // Document loading interfaces - for data ingestion + "document-load": { + "description": "Document loader", + "kind": "send", + "visible": true, + }, + "text-load": { + "description": "Text document loader", + "kind": "send", + "visible": true, + }, + + // Data storage interfaces - for processed data streams + "entity-contexts-load": { + "description": "Entity contexts loader", + "kind": "send", + }, + "triples-store": { + "description": "Triples loader", + "kind": "send", + }, + "graph-embeddings-store": { + "description": "Graph embeddings loader", + "kind": "send", + }, + "document-embeddings-store": { + "description": "Document embeddings loader", + "kind": "send", + }, + "objects-store": { + "description": "Object store", + "kind": "request-response", + }, + + // Query interfaces - for retrieving information + "graph-rag": { + "description": "GraphRAG service", + "kind": "request-response", + }, + "document-rag": { + "description": "ChunkRAG service", + "kind": "request-response", + }, + "triples": { + "description": "Triples query service", + "kind": "request-response", + }, + "graph-embeddings": { + "description": "Graph embeddings service", + "kind": "request-response", + }, + "document-embeddings": { + "description": "Document embeddings service", + "kind": "request-response", + }, + "objects": { + "description": "Object query service", + "kind": "request-response", + }, + + // Processing services - for text and data processing + "prompt": { + "description": "Prompt service", + "kind": "request-response", + }, + "agent": { + "description": "Agent service", + "kind": "request-response", + }, + "text-completion": { + "description": "Text completion service", + "kind": "request-response", + }, + + // Query translation services - for natural language queries + "nlp-query": { + "description": "NLP question to GraphQL service", + "kind": "request-response", + }, + "structured-query": { + "description": "Structured query service", + "kind": "request-response", + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/parameter-processor.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/parameter-processor.jsonnet new file mode 100644 index 00000000..f317ae9f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/parameter-processor.jsonnet @@ -0,0 +1,38 @@ +// Parameter Processing Module +// Handles dynamic parameter replacement in configuration values +// Replaces {parameter_name} placeholders with actual parameter values + +{ + // Applies parameter substitutions to string values + // Only processes strings - leaves other types unchanged + substitute_parameters: function(value, parameters) + if std.isString(value) then + std.foldl( + function(acc, param) + // Only do string replacement if param.value is a string + if std.isString(param.value) then + std.strReplace(acc, "{" + param.key + "}", param.value) + else + acc, // Skip replacement for non-string parameter values + std.objectKeysValuesAll(parameters), + value + ) + else + value, + + // Applies parameter substitutions to all values in an object + // Recursively processes nested objects and arrays + substitute_parameters_in_object: function(obj, parameters) + if std.isObject(obj) then + { + [key]: $.substitute_parameters_in_object(obj[key], parameters) + for key in std.objectFields(obj) + } + else if std.isArray(obj) then + [ + $.substitute_parameters_in_object(item, parameters) + for item in obj + ] + else + $.substitute_parameters(obj, parameters), +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/tools.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/tools.jsonnet new file mode 100644 index 00000000..4947279e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/tools.jsonnet @@ -0,0 +1,36 @@ +// Tools Configuration Module +// Defines all available tools that can be used by agents and flows +// Each tool specifies its interface, arguments, and behavior + +[ + // Knowledge query tool - queries the knowledge base + { + id: "knowledge-query", + name: "Knowledge query", + description: "This tool queries a knowledge base that holds information about domain-specific information. The question should be a natural language question.", + type: "knowledge-query", + collection: "default", + arguments: [ + { + name: "question", + type: "string", + description: "A simple natural language question.", + } + ] + }, + + // LLM completion tool - general purpose text completion + { + id: "llm-completion", + name: "LLM text completion", + type: "text-completion", + description: "This tool queries an LLM for non-domain-specific information. The question should be a natural language question.", + arguments: [ + { + name: "question", + type: "string", + description: "The question which should be asked of the LLM.", + } + ] + } +] \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/trustgraph-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/trustgraph-config.jsonnet new file mode 100644 index 00000000..d860ab3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/runtime-config/trustgraph-config.jsonnet @@ -0,0 +1,90 @@ +// TrustGraph Main Configuration +// Clean, modular composition of TrustGraph configuration +// Uses specialized modules for different aspects of config building + +// Import dependencies +local images = import "../values/images.jsonnet"; +local url = import "../values/url.jsonnet"; +local prompts = import "../prompts/mixtral.jsonnet"; +local default_prompts = import "../prompts/default-prompts.jsonnet"; +local token_costs = import "../values/token-costs.jsonnet"; +local flow_blueprints = import "../flows/flow-blueprints.jsonnet"; +local config_composer = import "config-composer.jsonnet"; +local interface_descriptions = import "interface-descriptions.jsonnet"; +local tools = import "tools.jsonnet"; +local temperature_params = import "../parameters/temperature-param-types.jsonnet"; +local chunking_params = import "../parameters/chunking-param-types.jsonnet"; + +// Main configuration object +local configuration = { + + // Prompt templates + prompts:: default_prompts, + + // Tool definitions + tools:: tools, + + // MCP configuration + mcp:: {}, + + // Flow classes reference + "flow-blueprints":: flow_blueprints, + + // LLM model parameters + "llm-models" +:: {}, + + // Embeddings model parameters + "embeddings-models" +:: {}, + + collections +:: { + "trustgraph:default": { + "user": "default-user", + "collection": "default", + "name": "Default Collection", + "description": "Default collection", + "tags": ["default"], + }, + }, + + // Default model and flow parameters + flow_init_parameters:: { + "llm-model": $["llm-models"].default, + "llm-rag-model": $["llm-models"].default, + "llm-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "llm-rag-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "chunk-size": std.toString($["parameter-types"]["chunk-size"].default), + "chunk-overlap": std.toString($["parameter-types"]["chunk-overlap"].default), + "embeddings-model": $["embeddings-models"].default, + }, + + // Interface descriptions for external endpoints + "interface-descriptions":: interface_descriptions, + + // Parameter type definitions + "parameter-types":: { + "llm-model": $["llm-models"], + "embeddings-model": $["embeddings-models"], + } + chunking_params + temperature_params, + + // Token costs + "token-costs":: token_costs, + + // Build the complete configuration using the composer + configuration:: config_composer.build({ + flow_blueprints: $["flow-blueprints"], + default_flow_blueprint: "everything", + default_flow_id: "default", + flow_init_parameters: $["flow_init_parameters"], + prompts: $["prompts"], + tools: $["tools"], + mcp: $["mcp"], + interface_descriptions: $["interface-descriptions"], + parameter_types: $["parameter-types"], + token_costs: $["token-costs"], + collection: $["collections"], + }), + +} + default_prompts; + +// Export the final configuration +configuration diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/cassandra.jsonnet new file mode 100644 index 00000000..05f4fd27 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/cassandra.jsonnet @@ -0,0 +1,77 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "backends/cassandra.jsonnet"; + +cassandra + { + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/falkordb.jsonnet new file mode 100644 index 00000000..b4bace2d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/falkordb.jsonnet @@ -0,0 +1,79 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local falkordb = import "backends/falkordb.jsonnet"; + +falkordb + { + + "falkordb-url":: "falkor://falkordb:6379", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/memgraph.jsonnet new file mode 100644 index 00000000..72597d72 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/memgraph.jsonnet @@ -0,0 +1,84 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local memgraph = import "backends/memgraph.jsonnet"; + +memgraph + { + + "memgraph-url":: "bolt://memgraph:7687", + "memgraph-database":: "memgraph", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/neo4j.jsonnet new file mode 100644 index 00000000..08904715 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/triple-store/neo4j.jsonnet @@ -0,0 +1,79 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local neo4j = import "backends/neo4j.jsonnet"; + +neo4j + { + + "neo4j-url":: "bolt://neo4j:7687", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/ui/workbench-ui.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/ui/workbench-ui.jsonnet new file mode 100644 index 00000000..f2048e47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/ui/workbench-ui.jsonnet @@ -0,0 +1,32 @@ +local images = import "values/images.jsonnet"; + +{ + + "workbench-ui" +: { + + create:: function(engine) + + local container = + engine.container("workbench-ui") + .with_image(images["workbench-ui"]) + .with_limits("0.1", "256M") + .with_reservations("0.1", "256M") + .with_port(8888, 8888, "ui"); + + local containerSet = engine.containers( + "workbench-ui", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8888, 8888, "ui"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/values/images.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/values/images.jsonnet new file mode 100644 index 00000000..8b8b75a0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/values/images.jsonnet @@ -0,0 +1,36 @@ +local version = import "version.jsonnet"; +{ + cassandra: "docker.io/cassandra:4.1.10", +// Not working +// ceph: "quay.io/ceph/daemon:latest-reef", + neo4j: "docker.io/neo4j:2025.08.0-community-bullseye", + pulsar: "docker.io/apachepulsar/pulsar:4.1.0", + pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.4.0", + etcd: "quay.io/coreos/etcd:v3.6.4", + minio: "docker.io/minio/minio:RELEASE.2025-09-07T16-13-09Z", + garage: "docker.io/dxflrs/garage:v2.1.0", + milvus: "docker.io/milvusdb/milvus:v2.5.17", + prometheus: "docker.io/prom/prometheus:v3.8.0", + grafana: "docker.io/grafana/grafana:12.3.0", + loki: "docker.io/grafana/loki:3.6.2", + trustgraph_base: "docker.io/trustgraph/trustgraph-base:" + version, + trustgraph_flow: "docker.io/trustgraph/trustgraph-flow:" + version, + trustgraph_ocr: "docker.io/trustgraph/trustgraph-ocr:" + version, + trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version, + trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version, + trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version, + trustgraph_mcp: "docker.io/trustgraph/trustgraph-mcp:" + version, + qdrant: "docker.io/qdrant/qdrant:v1.15.4", + memgraph_mage: "docker.io/memgraph/memgraph-mage:3.5", + memgraph_lab: "docker.io/memgraph/lab:3.5.0", + falkordb: "docker.io/falkordb/falkordb:v4.12.5", + "workbench-ui": "docker.io/trustgraph/workbench-ui:1.5.5", + "ddg-mcp-server": "docker.io/trustgraph/ddg-mcp-server:0.1.0", + "tgi-service-intel-xpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-xpu", + "tgi-service-cpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-cpu", + "tgi-service-gaudi": "ghcr.io/huggingface/text-generation-inference:sha-f140440-gaudi", + "vllm-service-intel-xpu": "docker.io/intel/vllm:0.8.0-xpu", + "vllm-service-gaudi": "docker.io/trustgraph/vllm-hpu:027f5645", + "vllm-service-nvidia": "docker.io/vllm/vllm-openai:latest", + "vllm-service-intel-battlemage": "docker.io/intelanalytics/ipex-llm-serving-xpu:0.2.0-b6", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/values/token-costs.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/values/token-costs.jsonnet new file mode 100644 index 00000000..84c70373 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/values/token-costs.jsonnet @@ -0,0 +1,102 @@ +{ + "mistral.mistral-large-2407-v1:0": { + "model_name": "mistral.mistral-large-2407-v1:0", + "input_price": 0.000004, + "output_price": 0.000012 + }, + "meta.llama3-1-405b-instruct-v1:0": { + "model_name": "meta.llama3-1-405b-instruct-v1:0", + "input_price": 0.00000532, + "output_price": 0.000016 + }, + "mistral.mixtral-8x7b-instruct-v0:1": { + "model_name": "mistral.mixtral-8x7b-instruct-v0:1", + "input_price": 0.00000045, + "output_price": 0.0000007 + }, + "meta.llama3-1-70b-instruct-v1:0": { + "model_name": "meta.llama3-1-70b-instruct-v1:0", + "input_price": 0.00000099, + "output_price": 0.00000099 + }, + "meta.llama3-1-8b-instruct-v1:0": { + "model_name": "meta.llama3-1-8b-instruct-v1:0", + "input_price": 0.00000022, + "output_price": 0.00000022 + }, + "anthropic.claude-3-haiku-20240307-v1:0": { + "model_name": "anthropic.claude-3-haiku-20240307-v1:0", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "anthropic.claude-3-5-sonnet-20240620-v1:0": { + "model_name": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "cohere.command-r-plus-v1:0": { + "model_name": "cohere.command-r-plus-v1:0", + "input_price": 0.0000030, + "output_price": 0.0000150 + }, + "ollama": { + "model_name": "ollama", + "input_price": 0, + "output_price": 0 + }, + "claude-3-haiku-20240307": { + "model_name": "claude-3-haiku-20240307", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "claude-3-5-sonnet-20240620": { + "model_name": "claude-3-5-sonnet-20240620", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "claude-3-opus-20240229": { + "model_name": "claude-3-opus-20240229", + "input_price": 0.000015, + "output_price": 0.000075 + }, + "claude-3-sonnet-20240229": { + "model_name": "claude-3-sonnet-20240229", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "command-r-08-202": { + "model_name": "command-r-08-202", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "c4ai-aya-23-8b": { + "model_name": "c4ai-aya-23-8b", + "input_price": 0, + "output_price": 0 + }, + "llama.cpp": { + "model_name": "llama.cpp", + "input_price": 0, + "output_price": 0 + }, + "gpt-4o": { + "model_name": "gpt-4o", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-2024-08-06": { + "model_name": "gpt-4o-2024-08-06", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "gpt-4o-2024-05-13": { + "model_name": "gpt-4o-2024-05-13", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-mini": { + "model_name": "gpt-4o-mini", + "input_price": 0.00000015, + "output_price": 0.0000006 + } +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/values/url.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/values/url.jsonnet new file mode 100644 index 00000000..c3d4ad97 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/values/url.jsonnet @@ -0,0 +1,7 @@ +{ + pulsar: "pulsar://pulsar:6650", + pulsar_admin: "http://pulsar:8080", + milvus: "http://milvus:19530", + qdrant: "http://qdrant:6333", + object_store: "garage:3900", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/vector-store/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/vector-store/milvus.jsonnet new file mode 100644 index 00000000..213027d1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/vector-store/milvus.jsonnet @@ -0,0 +1,146 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local milvus = import "backends/milvus.jsonnet"; + +milvus + { + + "store-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/vector-store/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/vector-store/pinecone.jsonnet new file mode 100644 index 00000000..3803bdcb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/vector-store/pinecone.jsonnet @@ -0,0 +1,160 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; + +{ + + "pinecone-cloud":: "aws", + "pinecone-region":: "us-east-1", + + "store-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/vector-store/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/vector-store/qdrant.jsonnet new file mode 100644 index 00000000..7409adb9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/vector-store/qdrant.jsonnet @@ -0,0 +1,169 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local qdrant = import "backends/qdrant.jsonnet"; + +qdrant + { + + "store-graph-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/zip-readme.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/zip-readme.md new file mode 100644 index 00000000..0b117792 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/1.9/zip-readme.md @@ -0,0 +1,28 @@ + +Note! this is a subset of possible configurations, to generate your own +launch config use the config util... + +- Production: https://config-ui.demo.trustgraph.ai +- Early release: https://dev.config-ui.demo.trustgraph.ai + +The config util auto-generates deployment instructions for your +configuration, so that's the recommended way to deploy. + +---------------------------------------------------------------------------- + +These are launch configurations for TrustGraph. See https://trustgraph.ai for +the quickstart using docker compose. + +Hint for Linux: There are files here which get mounted as volumes inside +Docker Compose containers. This may trigger SELinux rules on your system, to +permit access insider the containers, use a command like this... + +chcon -Rt svirt_sandbox_file_t grafana/ prometheus/ + +The file vertexai/private.json is a placeholder for real GCP credentials if +you are using the VertexAI LLM. If you're using that in Docker Compose, +replace with your real credentials, and don't forget to permit access if you +are using Linux: + +chcon -Rt svirt_sandbox_file_t vertexai/ + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/README.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/README.md new file mode 100644 index 00000000..23039e9a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/README.md @@ -0,0 +1,125 @@ + +# TrustGraph template generation + +There are two utilities here: + +- `generate`: Generates a single Docker Compose launch configuration + based on configuration you provide. +- `generate-all`: Generates the release bundle for releases. You won't + need to use this unless you are managing releases. + +## `generate-all` + +Previously, this generates a full set of all vector DB / triple store / LLM +combinations, and put them in a single ZIP file. But this got out of +hand, so at the time of writing, this generates a single configuraton +using Qdrant vector DB, Ollama LLM support and Cassandra for a triple store. + +The combinations are contained withing the code, it takes two arguments: +- output ZIP file (is over-written) +- TrustGraph version number + +``` +templates/generate-all output.zip 0.18.11 +``` + +## `generate` + +This utility takes a configuration file describing the components to bundle, +and outputs a Docker Compose YAML file. + +### Input configuration + +The input configuration is a JSON file, an array of components to pull into +the configuration. For each component, there is a name and a (possibly empty) +object describing addtional parameters for that component. + +Example: + +``` +[ + { + "name": "cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "qdrant", + "parameters": {} + }, + { + "name": "embeddings-hf", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph", + "parameters": {} + }, + { + "name": "googleaistudio", + "parameters": { + "googleaistudio-temperature": 0.3, + "googleaistudio-max-output-tokens": 2048, + "googleaistudio-model": "gemini-1.5-pro-002" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 1000, + "chunk-overlap": 50 + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "agent-manager-react", + "parameters": {} + } +] +``` + +If you want to make your own configuration you could try changing the +configuration above: +- Components which are essential: pulsar, trustgraph, graph-rag, grafana, + agent-manager-react +- You need a triple store, one of: cassandra, memgraph, falkordb, neo4j +- You need a vector store, one of: qdrant, pinecone +- You need an LLM, one of: azure, azure-openai, bedrock, claude, cohere, + llamafile, ollama, openai, vertexai. +- You need an embeddings implementation, one of: embeddings-hf, + embeddings-ollama +- Optionally add the Workbench tool: workbench-ui + +Components have over-ridable parameters, look in the component definition +in `templates/components/` to see what you can override. + +### Invocation + +Two parameters: +- The output ZIP file +- The version number + +The configuration file described above is provided on standard input + +``` +templates/generate out.zip 0.18.9 < config.json +``` + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/cassandra.jsonnet new file mode 100644 index 00000000..60f262d3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/cassandra.jsonnet @@ -0,0 +1,50 @@ +local images = import "values/images.jsonnet"; + +{ + + "cassandra" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1000M", + "memory-reservation":: "1000M", + "heap":: "300M", + + create:: function(engine) + + // Capture memory settings into locals + local memLimit = self["memory-limit"]; + local memReserv = self["memory-reservation"]; + local heap = self["heap"]; + + local vol = engine.volume("cassandra").with_size("20G"); + + local container = + engine.container("cassandra") + .with_image(images.cassandra) + .with_environment({ + JVM_OPTS: "-Xms%s -Xmx%s -Dcassandra.skip_wait_for_gossip_to_settle=0" % [ + heap, heap, + ], + }) + .with_limits("1.0", memLimit) + .with_reservations("0.5", memReserv) + .with_port(9042, 9042, "cassandra") + .with_volume_mount(vol, "/var/lib/cassandra"); + + local containerSet = engine.containers( + "cassandra", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9042, 9042, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/ceph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/ceph.jsonnet new file mode 100644 index 00000000..42406a95 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/ceph.jsonnet @@ -0,0 +1,446 @@ + +// +// This majorly does not work +// + +// This configuration fails primarily because it tries to treat Ceph +// like a stateless web app. You are currently pointing mon host to a +// generic service name (ceph-mon), but you aren't telling the Monitor +// process to assume that service identity. + +// To make this work with the "Service Name" approach across any +// engine, you need to fix the binding logic and the messenger protocol. + +// 1. The ceph.conf Fix + +// Need to enable Messenger v2 (modern) and tell the cluster to use +// the service names for its initial quorum. + +//Change this section: +// Ini, TOML + +// [global] +// fsid = %s +// mon initial members = mon0 +// mon host = ceph-mon:6789 +// ... + +// To this: +// Ini, TOML + +// [global] +// fsid = %s +// # Use the actual service names as members +// mon initial members = ceph-mon +// # Explicitly use the Service name (VIP) +// mon host = ceph-mon +// # Force modern protocol +// ms_bind_msgr2 = true +// ms_bind_msgr1 = true + +// 2. The MON Environment & Command Fix + +// This is the most critical part. Your current config sets MON_IP: +// "0.0.0.0". This causes the MON to bind to the Pod IP, which breaks +// when the Pod restarts. You must force it to bind to the Service IP. + +// Update mon_env: +// Code snippet + +// local mon_env = cluster_env + { +// CEPH_DAEMON: "MON", +// MON_NAME: "mon0", +// # Remove MON_IP: "0.0.0.0" +// # Add these: +// MON_ADDR: "ceph-mon", // This says to resolve the service name +// }; + +// Update mon_container command: You are currently wiping the MON data +// on every start (rm -rf /var/lib/ceph/mon/*). Stop doing that. If you +// wipe the data, you lose the cluster state and the OSDs will refuse to +// talk to the "new" MON. + +// Code snippet + +// .with_command([ +// "bash", "-c", +// # 1. Resolve the Service IP at runtime +// "export MON_IP=$(getent hosts ceph-mon | awk '{ print $1 }'); " + +// inject_mon_config + +// # 2. Start the daemon telling it its PUBLIC address is the Service VIP +// "exec /opt/ceph-container/bin/entrypoint.sh" +// ]) + +// 3. Why your current config "Majorly does not work" + +// The "Wipe" Logic: By running rm -rf /var/lib/ceph/mon/* in the +// MON container, you are creating a "New Cluster" every time the +// container starts. Since the OSDs store the fsid and cluster +// secrets, they will reject the "new" MON. + +// DNS Race Condition: Your OSD/MGR/RGW containers wait for +// ceph-mon DNS, which is good. However, if ceph-mon resolves to a +// Round Robin IP (multiple pods) rather than a stable ClusterIP, the +// connection will be flaky. + +// Messenger Protocol: Without ms_bind_msgr2, Ceph defaults to the +// old v1 protocol which is much more sensitive to NAT/Container IP +// mismatches. + +// SUMMARY + +// Component: ceph.conf +// Change: Add ms_bind_msgr2 = true +// Why: Supports modern container networking better. + +// Component: MON Start +// Change: Remove rm -rf +// Why: Ceph MONs must keep their database to maintain the cluster. + +// Component: MON Address +// Change: Use getent hosts ceph-mon +// Why: Forces the MON to advertise the Service VIP instead of its own Pod IP. + +// Component: MON Keyring +// Change: Ensure /etc/ceph/ceph.mon.keyring exists +// Why: MONs need their specific key to start. + +local images = import "values/images.jsonnet"; + +{ + with:: function(key, value) + self + { + ["ceph-" + key]:: value, + }, + + // Ceph credentials and cluster settings + "ceph-access-key":: "object-user", + "ceph-secret-key":: "object-password", + "ceph-cluster-id":: "ceph", + "ceph-fsid":: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + + // Pool redundancy settings + // size: 2 = two replicas for fault tolerance + // min_size: 1 = allow degraded I/O if one OSD is down (prevents cluster freeze) + "ceph-pool-size":: "2", + "ceph-pool-min-size":: "1", + + ceph +: { + create:: function(engine) + // Pre-Shared Cryptographic Material - Config-as-Code Approach + // These keys are generated once and distributed to all daemons + // This ensures cryptographic consistency across the shared-nothing architecture + local admin_key = "AQBpxSBlAAAAABAAU99V6D8vS7Uu9y1S8W0iBg=="; + local mon_key = "AQBpxSBlAAAAABAAn7pL/pG9oT+X6vO7V1S6bg=="; + + // Ceph configuration file - rendered from Jsonnet variables + local ceph_conf = ||| + [global] + fsid = %s + mon initial members = mon0 + mon host = ceph-mon:6789 + public network = 0.0.0.0/0 + cluster network = 0.0.0.0/0 + osd pool default size = %s + osd pool default min size = %s + osd crush chooseleaf type = 0 + auth cluster required = cephx + auth service required = cephx + auth client required = cephx + ||| % [$["ceph-fsid"], $["ceph-pool-size"], $["ceph-pool-min-size"]]; + + // Admin keyring - distributed to all daemons + local admin_keyring = ||| + [client.admin] + key = %s + caps mds = "allow *" + caps mgr = "allow *" + caps mon = "allow *" + caps osd = "allow *" + ||| % [admin_key]; + + // Monitor keyring - used by MON for cluster operations + local mon_keyring = ||| + [mon.] + key = %s + caps mon = "allow *" + ||| % [mon_key]; + + // Config injection command - writes files before entrypoint + local inject_config = "printf '%s' > /etc/ceph/ceph.conf; printf '%s' > /etc/ceph/ceph.client.admin.keyring; " % [ceph_conf, admin_keyring]; + local inject_mon_config = inject_config + ("printf '%s' > /etc/ceph/ceph.mon.keyring; " % [mon_keyring]); + + // Data volumes - sized appropriately for production workloads + local vol_mon = engine.volume("ceph-mon").with_size("20G"); + local vol_mgr = engine.volume("ceph-mgr").with_size("20G"); + local vol_osd = engine.volume("ceph-osd").with_size("100G"); + local vol_rgw = engine.volume("ceph-rgw").with_size("20G"); + + // Isolated config volumes per daemon (ReadWriteOnce compatible) + // Each daemon gets its own non-shared config volume to support + // multi-node scheduling in K8s and other orchestrators + local vol_mon_config = engine.volume("ceph-mon-config").with_size("500M"); + local vol_mgr_config = engine.volume("ceph-mgr-config").with_size("500M"); + local vol_osd_config = engine.volume("ceph-osd-config").with_size("500M"); + local vol_rgw_config = engine.volume("ceph-rgw-config").with_size("500M"); + local vol_init_config = engine.volume("ceph-init-config").with_size("500M"); + + // Simplified cluster environment - Config-as-Code model + // No fetch logic needed - config is injected before entrypoint runs + local cluster_env = { + CLUSTER: $["ceph-cluster-id"], + FSID: $["ceph-fsid"], + KV_TYPE: "none", // No external coordination + }; + + // MON-specific environment + // Config-as-Code: MON uses injected config files, not fetch logic + // + // CRITICAL: MON_DATA_AVAIL="0" forces fresh cluster bootstrap + // The ceph/daemon entrypoint script (variables_stack.sh) uses this as a gate: + // - MON_DATA_AVAIL="0" -> run mkfs, create new cluster with our FSID + // - MON_DATA_AVAIL="1" -> attempt to join existing cluster (infinite probe loop) + // + // Network configuration for monmap generation + local mon_env = cluster_env + { + CEPH_DAEMON: "MON", + MON_NAME: "mon0", + MON_PORT: "6789", + MON_DATA_AVAIL: "0", + MON_IP: "0.0.0.0", + NETWORK_AUTO_DETECT: "4", + CEPH_PUBLIC_NETWORK: "0.0.0.0/0", + }; + + // Simplified daemon environments - Config-as-Code model + // All daemons receive config via injection, not fetch from MON + // This eliminates "static mode" errors and networking complexity + + // MGR-specific environment + local mgr_env = cluster_env + { + CEPH_DAEMON: "MGR", + MGR_NAME: "mgr0", + }; + + // OSD-specific environment + local osd_env = cluster_env + { + CEPH_DAEMON: "OSD", + OSD_TYPE: "directory", + }; + + // RGW-specific environment + local rgw_env = cluster_env + { + CEPH_DAEMON: "RGW", + RGW_NAME: "rgw0", + RGW_FRONTEND_PORT: "7480", + }; + + // MON (Monitor) container - cluster state and quorum + // Config-as-Code: Injects pre-shared keys before entrypoint + // CRITICAL: Wipes /var/lib/ceph/mon/* on every start to force fresh bootstrap + // This ensures MON always uses our FSID and doesn't inherit stale cluster state + local mon_container = + engine.container("ceph-mon") + .with_image(images.ceph) + .with_environment(mon_env) + .with_command([ + "bash", "-c", + "rm -rf /var/lib/ceph/mon/*; " + + inject_mon_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(6789, 6789, "mon") + .with_port(3300, 3300, "mon-msgr2") + .with_volume_mount(vol_mon, "/var/lib/ceph/mon") + .with_volume_mount(vol_mon_config, "/etc/ceph"); + + // MGR (Manager) container - cluster management and dashboard + // Config-as-Code: Uses injected config files with pre-shared keys + // DNS wait ensures MON is available before MGR connects + local mgr_container = + engine.container("ceph-mgr") + .with_image(images.ceph) + .with_environment(mgr_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(7000, 7000, "mgr") + .with_port(8443, 8443, "dashboard") + .with_port(9283, 9283, "prometheus") + .with_volume_mount(vol_mgr, "/var/lib/ceph/mgr") + .with_volume_mount(vol_mgr_config, "/etc/ceph"); + + // OSD (Object Storage Daemon) - actual data storage + // Config-as-Code: Uses injected config files with pre-shared keys + // Increased resources to prevent OOM during recovery operations + // DNS wait ensures MON is available before OSD connects + local osd_container = + engine.container("ceph-osd") + .with_image(images.ceph) + .with_environment(osd_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("2.0", "4096M") + .with_reservations("1.0", "2048M") + .with_port(6800, 6800, "osd") + .with_volume_mount(vol_osd, "/var/lib/ceph/osd") + .with_volume_mount(vol_osd_config, "/etc/ceph"); + + // RGW (RADOS Gateway) - S3 API endpoint + // Config-as-Code: Uses injected config files with pre-shared keys + // DNS wait ensures MON is available before RGW connects + local rgw_container = + engine.container("ceph-rgw") + .with_image(images.ceph) + .with_environment(rgw_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(7480, 7480, "s3") + .with_volume_mount(vol_rgw, "/var/lib/ceph/radosgw") + .with_volume_mount(vol_rgw_config, "/etc/ceph"); + + // Init container - one-time S3 user provisioning + // IMPORTANT: This container exits with code 0 after completion + // Orchestrator must NOT restart it (use K8s Job or Compose restart: "no") + // Config-as-Code: Uses injected config to run radosgw-admin commands + local init_container = + engine.container("ceph-init") + .with_image(images.ceph) + .with_environment({ + CLUSTER: $["ceph-cluster-id"], + FSID: $["ceph-fsid"], + KV_TYPE: "none", + RGW_ACCESS_KEY: $["ceph-access-key"], + RGW_SECRET_KEY: $["ceph-secret-key"], + }) + .with_limits("0.5", "512M") + .with_reservations("0.25", "256M") + .with_volume_mount(vol_init_config, "/etc/ceph") + .with_command([ + "bash", "-c", + inject_config + ||| + set -e + + # Wait for cluster health + echo "Waiting for Ceph cluster to be healthy..." + MAX_ATTEMPTS=60 + ATTEMPT=0 + until ceph --cluster ${CLUSTER} health 2>/dev/null | grep -q "HEALTH_OK\|HEALTH_WARN"; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: Cluster failed to become healthy after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: Cluster not ready, retrying in 5s..." + sleep 5 + done + echo "Cluster is healthy." + + # Wait for RGW availability + echo "Waiting for RGW to be ready..." + ATTEMPT=0 + until curl -sf http://ceph-rgw:7480 >/dev/null 2>&1; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: RGW failed to become ready after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: RGW not ready, retrying in 5s..." + sleep 5 + done + echo "RGW is ready." + + # Idempotent S3 user creation + echo "Provisioning S3 user: ${RGW_ACCESS_KEY}" + if radosgw-admin --cluster ${CLUSTER} user info --uid="${RGW_ACCESS_KEY}" >/dev/null 2>&1; then + echo "User ${RGW_ACCESS_KEY} already exists, skipping creation." + else + echo "Creating new S3 user: ${RGW_ACCESS_KEY}" + radosgw-admin --cluster ${CLUSTER} user create \ + --uid="${RGW_ACCESS_KEY}" \ + --display-name="Object Storage User" \ + --access-key="${RGW_ACCESS_KEY}" \ + --secret-key="${RGW_SECRET_KEY}" + echo "S3 user created successfully." + fi + + echo "Initialization complete. Exiting." + exit 0 + |||, + ]); + + // Container sets - each daemon gets its own for K8s node distribution + local mon_containerSet = engine.containers("ceph-mon", [mon_container]); + local mgr_containerSet = engine.containers("ceph-mgr", [mgr_container]); + local osd_containerSet = engine.containers("ceph-osd", [osd_container]); + local rgw_containerSet = engine.containers("ceph-rgw", [rgw_container]); + local init_containerSet = engine.containers("ceph-init", [init_container]); + + // Services - expose daemon ports for inter-daemon communication + local mon_service = + engine.service(mon_containerSet) + .with_port(6789, 6789, "mon") + .with_port(3300, 3300, "mon-msgr2"); + + local mgr_service = + engine.service(mgr_containerSet) + .with_port(7000, 7000, "mgr") + .with_port(8443, 8443, "dashboard") + .with_port(9283, 9283, "prometheus"); + + local osd_service = + engine.service(osd_containerSet) + .with_port(6800, 6800, "osd"); + + local rgw_service = + engine.service(rgw_containerSet) + .with_port(7480, 7480, "s3"); + + engine.resources([ + + // Data volumes + vol_mon, + vol_mgr, + vol_osd, + vol_rgw, + + // Config volumes (isolated, no sharing) + vol_mon_config, + vol_mgr_config, + vol_osd_config, + vol_rgw_config, + vol_init_config, + + // Container sets + mon_containerSet, + mgr_containerSet, + osd_containerSet, + rgw_containerSet, + init_containerSet, + + // Services + mon_service, + mgr_service, + osd_service, + rgw_service, + + ]) + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/falkordb.jsonnet new file mode 100644 index 00000000..1d4176d8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/falkordb.jsonnet @@ -0,0 +1,38 @@ +local images = import "values/images.jsonnet"; + +{ + + "falkordb" +: { + + create:: function(engine) + + local vol = engine.volume("falkordb").with_size("20G"); + + local container = + engine.container("falkordb") + .with_image(images.falkordb) + .with_limits("1.0", "768M") + .with_reservations("0.5", "768M") + .with_port(6379, 6379, "api") + .with_port(3010, 3000, "ui") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "falkordb", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6379, 6379, "api") + .with_port(3010, 3010, "ui"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/garage.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/garage.jsonnet new file mode 100644 index 00000000..9d339bfb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/garage.jsonnet @@ -0,0 +1,249 @@ +local images = import "values/images.jsonnet"; + +{ + + garage +: { + + // Garage S3 credentials - these are the actual access key ID and secret key + // Access Key ID must be in format: GK + 24 hex characters (12 bytes) + // Secret Key must be 64 hex characters (32 bytes) + // For production, generate secure random values and override these defaults + "access-key":: "GK000000000000000000000001", + "secret-key":: "b171f00be9be4c32c734f4c05fe64c527a8ab5eb823b376cfa8c2531f70fc427", + "rpc-secret":: "bbba746a9e289bad64a9e7a36a4299dac8d6e0b8cc2a6c2937fe756df4492008", + // For a production system, override this value + "admin-token":: "batts-rockhearted-unpartially", + region:: "garage", + "replication-factor":: "1", // Set to 1 for single-node, 3 for production + + // Storage volume sizes + "meta-size":: "2G", // Metadata volume size + "data-size":: "5G", // Data volume size (also used for cluster layout capacity) + + create:: function(engine) + + local accessKey = self["access-key"]; + local secretKey = self["secret-key"]; + local rpcSecret = self["rpc-secret"]; + local adminToken = self["admin-token"]; + local region = self.region; + local replicationFactor = self["replication-factor"]; + local metaSize = self["meta-size"]; + local dataSize = self["data-size"]; + + // Garage daemon configuration file - TOML format + local garage_conf = ||| + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + + db_engine = "lmdb" + + replication_factor = %s + + compression_level = 1 + + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "[::]:3901" + rpc_secret = "%s" + + [s3_api] + s3_region = "%s" + api_bind_addr = "[::]:3900" + root_domain = ".s3.garage.local" + + [s3_web] + bind_addr = "[::]:3902" + root_domain = ".web.garage.local" + index = "index.html" + + [k2v_api] + api_bind_addr = "[::]:3904" + + [admin] + api_bind_addr = "[::]:3903" + admin_token = "%s" + ||| % [replicationFactor, rpcSecret, region, adminToken]; + + // Config volume - contains the rendered garage.toml + local cfgVol = engine.configVolume( + "garage-cfg", "garage", + { + "garage.toml": garage_conf, + } + ); + + // Volumes - Garage stores metadata and data separately + local vol_meta = engine.volume("garage-meta").with_size(metaSize); + local vol_data = engine.volume("garage-data").with_size(dataSize); + + // Main Garage daemon container + local garage_container = + engine.container("garage") + .with_image(images.garage) + .with_command([ + "/garage", "-c", "/etc/garage/garage.toml", "server" + ]) + .with_environment({ + RUST_LOG: "garage=info", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3900, 3900, "s3-api") + .with_port(3901, 3901, "rpc") + .with_port(3902, 3902, "web") + .with_port(3903, 3903, "admin") + .with_port(3904, 3904, "k2v") + .with_volume_mount(cfgVol, "/etc/garage/") + .with_volume_mount(vol_meta, "/var/lib/garage/meta") + .with_volume_mount(vol_data, "/var/lib/garage/data"); + + // Init container - configures cluster layout and creates S3 credentials + // IMPORTANT: This container exits with code 0 after completion + // Orchestrator must NOT restart it (use K8s Job or Compose restart: "no") + // Uses Alpine base image since garage container has no shell + local init_container = + engine.container("garage-init") + .with_image("docker.io/alpine:3.23.2") + .with_environment({ + GARAGE_ACCESS_KEY: accessKey, + GARAGE_SECRET_KEY: secretKey, + GARAGE_REGION: region, + GARAGE_ADMIN_TOKEN: adminToken, + GARAGE_RPC_SECRET: rpcSecret, + GARAGE_DATA_SIZE: dataSize, + }) + .with_limits("0.5", "256M") + .with_reservations("0.25", "128M") + .with_volume_mount(cfgVol, "/etc/garage/") + .with_command([ + "sh", "-c", ||| + set -e + + # Install required tools + echo "Installing curl, jq and downloading garage CLI..." + apk add --no-cache curl jq + + # Download garage binary (v2.1.0) for remote management + curl -fsSL "https://garagehq.deuxfleurs.fr/_releases/v2.1.0/x86_64-unknown-linux-musl/garage" \ + -o /usr/local/bin/garage + chmod +x /usr/local/bin/garage + + echo "Waiting for Garage daemon to be ready..." + MAX_ATTEMPTS=60 + ATTEMPT=0 + # Wait for /health to respond (even 503 is fine - means daemon is up) + until curl -s http://garage:3903/health >/dev/null 2>&1; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: Garage failed to become ready after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: Garage not ready, retrying in 2s..." + sleep 2 + done + echo "Garage daemon is ready." + + # Get the node ID via v2 Admin API + echo "Getting Garage node ID via Admin API..." + curl -s -H "Authorization: Bearer ${GARAGE_ADMIN_TOKEN}" \ + "http://garage:3903/v2/GetNodeInfo?node=self" > /tmp/garage-node-info.json + + # Extract node ID from response (the key in success map is the node ID) + NODE_ID=$(jq -r '.success | to_entries[0].value.nodeId' /tmp/garage-node-info.json) + echo "Node ID: ${NODE_ID}" + + if [ -z "$NODE_ID" ] || [ "$NODE_ID" = "null" ]; then + echo "ERROR: Failed to retrieve node ID" + exit 1 + fi + + # ===== LAYOUT MANAGEMENT VIA REMOTE RPC ===== + # Use garage CLI with -h and -s flags to connect remotely + # -h requires format: @: + # -s provides the RPC secret + + # Construct full RPC identifier + RPC_HOST="${NODE_ID}@garage:3901" + echo "RPC Host: ${RPC_HOST}" + + # Check current layout to see if node is already assigned (idempotent) + echo "Checking current cluster layout..." + LAYOUT_OUTPUT=$(garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" layout show 2>&1) + + # Check if node already has a role assigned + # Layout output shows abbreviated node ID (first 16 chars) + NODE_ID_SHORT="${NODE_ID:0:16}" + if echo "$LAYOUT_OUTPUT" | grep -q "$NODE_ID_SHORT"; then + echo "Node ${NODE_ID_SHORT}... already assigned in layout, skipping." + else + echo "Assigning node to cluster layout..." + # Assign node to zone dc1 with configured capacity + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + layout assign ${NODE_ID} -z dc1 -c ${GARAGE_DATA_SIZE} + + echo "Applying layout configuration..." + # Get current staged version and apply + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + layout apply --version 1 + + echo "Layout configured successfully." + # Wait for layout to stabilize + sleep 5 + fi + + # ===== KEY MANAGEMENT VIA REMOTE RPC ===== + + # Check if key already exists (idempotent) + # GARAGE_ACCESS_KEY is already a valid Garage Key ID (GK + 24 hex chars) + if garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" key info "${GARAGE_ACCESS_KEY}" >/dev/null 2>&1; then + echo "Access key ${GARAGE_ACCESS_KEY} already exists, skipping creation." + else + echo "Importing S3 access key: ${GARAGE_ACCESS_KEY}" + + # Import key with the provided credentials (both already in valid Garage format) + # GARAGE_ACCESS_KEY = Key ID (GK + 24 hex chars) + # GARAGE_SECRET_KEY = Secret (64 hex chars) + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + key import "${GARAGE_ACCESS_KEY}" "${GARAGE_SECRET_KEY}" --yes + + echo "Access key imported successfully." + fi + + # Grant permissions to the key + echo "Granting create-bucket permission to key..." + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + key allow --create-bucket "${GARAGE_ACCESS_KEY}" + + echo "" + echo "Garage initialization complete!" + echo "S3 Endpoint: http://garage:3900" + echo "Region: ${GARAGE_REGION}" + exit 0 + |||, + ]); + + // Container sets + local garage_containerSet = engine.containers("garage", [garage_container]); + local init_containerSet = engine.containers("garage-init", [init_container]); + + // Service - expose Garage ports + local garage_service = + engine.service(garage_containerSet) + .with_port(3900, 3900, "s3-api") + .with_port(3901, 3901, "rpc") + .with_port(3902, 3902, "web") + .with_port(3903, 3903, "admin") + .with_port(3904, 3904, "k2v"); + + engine.resources([ + cfgVol, + vol_meta, + vol_data, + garage_containerSet, + init_containerSet, + garage_service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/memgraph.jsonnet new file mode 100644 index 00000000..eeed2e4e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/memgraph.jsonnet @@ -0,0 +1,70 @@ +local images = import "values/images.jsonnet"; + +{ + + "memgraph" +: { + + create:: function(engine) + + local vol = engine.volume("memgraph").with_size("20G"); + + local container = + engine.container("memgraph") + .with_image(images.memgraph_mage) + .with_environment({ + MEMGRAPH: "--storage-properties-on-edges=true --storage-enable-edges-metadata=true" + }) + .with_limits("1.0", "1000M") + .with_reservations("0.5", "1000M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/var/lib/memgraph"); + + local containerSet = engine.containers( + "memgraph", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + "memgraph-lab" +: { + + create:: function(engine) + + local container = + engine.container("lab") + .with_image(images.memgraph_lab) + .with_environment({ + QUICK_CONNECT_MG_HOST: "memgraph", + QUICK_CONNECT_MG_PORT: "7687", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3010, 3000, "http"); + + local containerSet = engine.containers( + "lab", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3010, 3010, "http"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/milvus.jsonnet new file mode 100644 index 00000000..5bed3820 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/milvus.jsonnet @@ -0,0 +1,89 @@ +local images = import "values/images.jsonnet"; +local minio = import "backends/minio.jsonnet"; + +minio { + + etcd +: { + + create:: function(engine) + + local vol = engine.volume("etcd").with_size("20G"); + + local container = + engine.container("etcd") + .with_image(images.etcd) + .with_command([ + "etcd", + "-advertise-client-urls=http://127.0.0.1:2379", + "-listen-client-urls", + "http://0.0.0.0:2379", + "--data-dir", + "/etcd", + ]) + .with_environment({ + ETCD_AUTO_COMPACTION_MODE: "revision", + ETCD_AUTO_COMPACTION_RETENTION: "1000", + ETCD_QUOTA_BACKEND_BYTES: "4294967296", + ETCD_SNAPSHOT_COUNT: "50000" + }) + .with_limits("1.0", "128M") + .with_reservations("0.25", "128M") + .with_port(2379, 2379, "api") + .with_volume_mount(vol, "/etcd"); + + local containerSet = engine.containers( + "etcd", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(2379, 2379, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + milvus +: { + + create:: function(engine) + + local vol = engine.volume("milvus").with_size("20G"); + + local container = + engine.container("milvus") + .with_image(images.milvus) + .with_command([ + "milvus", "run", "standalone" + ]) + .with_environment({ + ETCD_ENDPOINTS: "etcd:2379", + MINIO_ADDRESS: "minio:9000", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2") + .with_volume_mount(vol, "/var/lib/milvus"); + + local containerSet = engine.containers( + "milvus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/minio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/minio.jsonnet new file mode 100644 index 00000000..b38bb81f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/minio.jsonnet @@ -0,0 +1,48 @@ +local images = import "values/images.jsonnet"; + +{ + + minio +: { + + create:: function(engine) + + local vol = engine.volume("minio-data").with_size("20G"); + + local container = + engine.container("minio") + .with_image(images.minio) + .with_command([ + "minio", + "server", + "/minio_data", + "--console-address", + ":9001", + ]) + .with_environment({ + MINIO_ROOT_USER: "minioadmin", + MINIO_ROOT_PASSWORD: "minioadmin", + }) + .with_limits("0.5", "128M") + .with_reservations("0.25", "128M") + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console") + .with_volume_mount(vol, "/minio_data"); + + local containerSet = engine.containers( + "minio", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/neo4j.jsonnet new file mode 100644 index 00000000..46c61e0f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/neo4j.jsonnet @@ -0,0 +1,46 @@ +local images = import "values/images.jsonnet"; + +{ + + "neo4j" +: { + + create:: function(engine) + + local vol = engine.volume("neo4j").with_size("20G"); + + local container = + engine.container("neo4j") + .with_image(images.neo4j) + .with_environment({ + NEO4J_AUTH: "neo4j/password", + NEO4J_server_memory_pagecache_size: "512m", + NEO4J_server_memory_heap_max__size: "512m", + // NEO4J_server_bolt_listen__address: "0.0.0.0:7687", + // NEO4J_server_default__listen__address: "0.0.0.0", + // NEO4J_server_http_listen__address: "0.0.0.0:7474", + }) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1536M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "neo4j", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/pinecone.jsonnet new file mode 100644 index 00000000..2bef70fb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/pinecone.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "pinecone" +: { + + create:: function(engine) + + local container = + engine.container("pinecone") + .with_image(images.pinecone) + .with_environment({ + + PORT: "5080", + + INDEX_TYPE: "serverless", + + // Dimension is fixed, 384 is right for miniLM + // sentence embedding + DIMENSION: 384, + + METRIC: "cosine" + + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(5080, 5080, "api"); + + local containerSet = engine.containers( + "pinecone", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(5080, 5080, "api"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/qdrant.jsonnet new file mode 100644 index 00000000..2e6761f8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/backends/qdrant.jsonnet @@ -0,0 +1,65 @@ +local images = import "values/images.jsonnet"; + +{ + + "qdrant" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1024M", + "memory-reservation":: "1024M", + + // Mmap settings for low-memory mode (trades latency for memory) + // Set to null to disable, or string value to enable + "memmap-threshold-kb":: null, + "on-disk-payload":: null, + + create:: function(engine) + + // Capture memory settings into locals + local memLimit = self["memory-limit"]; + local memReserv = self["memory-reservation"]; + local mmapThreshold = self["memmap-threshold-kb"]; + local onDiskPayload = self["on-disk-payload"]; + + local vol = engine.volume("qdrant").with_size("20G"); + + // Build environment with optional mmap settings + local baseEnv = {}; + local env = baseEnv + + (if mmapThreshold != null then { + QDRANT__STORAGE__MEMMAP_THRESHOLD_KB: mmapThreshold, + } else {}) + + (if onDiskPayload != null then { + QDRANT__STORAGE__ON_DISK_PAYLOAD: onDiskPayload, + } else {}); + + local container = + engine.container("qdrant") + .with_image(images.qdrant) + .with_limits("1.0", memLimit) + .with_reservations("0.5", memReserv) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2") + .with_volume_mount(vol, "/qdrant/storage") + + (if std.length(env) > 0 then { + environment+: env, + } else {}); + + local containerSet = engine.containers( + "qdrant", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/components.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/components.jsonnet new file mode 100644 index 00000000..f6157326 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/components.jsonnet @@ -0,0 +1,121 @@ +{ + + // Essentials + "trustgraph-base": import "core/trustgraph.jsonnet", + "rev-gateway": import "core/rev-gateway.jsonnet", + "pulsar": import "pulsar/pulsar.jsonnet", + + // LLMs + "azure": import "llm/azure.jsonnet", + "azure-openai": import "llm/azure-openai.jsonnet", + "bedrock": import "llm/bedrock.jsonnet", + "claude": import "llm/claude.jsonnet", + "cohere": import "llm/cohere.jsonnet", + "googleaistudio": import "llm/googleaistudio.jsonnet", + "llamafile": import "llm/llamafile.jsonnet", + "lmstudio": import "llm/lmstudio.jsonnet", + "mistral": import "llm/mistral.jsonnet", + "ollama": import "llm/ollama.jsonnet", + "openai": import "llm/openai.jsonnet", + "vertexai": import "llm/vertexai.jsonnet", + "tgi": import "llm/tgi.jsonnet", + "vllm": import "llm/vllm.jsonnet", + + // Embeddings + "embeddings-ollama": import "embeddings/embeddings-ollama.jsonnet", + "embeddings-hf": import "embeddings/embeddings-hf.jsonnet", + "embeddings-fastembed": import "embeddings/embeddings-fastembed.jsonnet", + + // OCR options + "ocr": import "ocr/ocr.jsonnet", + "mistral-ocr": import "ocr/mistral-ocr.jsonnet", + + // Vector stores + "vector-store-milvus": import "vector-store/milvus.jsonnet", + "vector-store-qdrant": import "vector-store/qdrant.jsonnet", + "vector-store-pinecone": import "vector-store/pinecone.jsonnet", + + // Triples stores + "triple-store-cassandra": import "triple-store/cassandra.jsonnet", + "triple-store-neo4j": import "triple-store/neo4j.jsonnet", + "triple-store-falkordb": import "triple-store/falkordb.jsonnet", + "triple-store-memgraph": import "triple-store/memgraph.jsonnet", + + // Row stores + "row-store-cassandra": import "row-store/cassandra.jsonnet", + + // Observability support + "grafana": import "monitoring/grafana.jsonnet", + "loki": import "monitoring/loki.jsonnet", + + // Pulsar manager is a UI for Pulsar. Uses a LOT of memory + "pulsar-manager": import "pulsar/pulsar-manager.jsonnet", + + "override-recursive-chunker": import "core/chunker-recursive.jsonnet", + + // The prompt manager + "prompt-overrides": import "core/prompt-overrides.jsonnet", + + // Extra MCP services + "ddg-mcp-server": import "mcp/ddg-mcp-server.jsonnet", + + // Does nothing. But, can be a hack to overwrite parameters + "null": {}, + + // Passthrough: returns parameters directly, preserving +: merge syntax + // Also supports JSON-safe routing with prefixed parameters like "cassandra-heap" + "override": { + local route = function(target) + function(prefix, k, v) + local suffix = std.substr(k, std.length(prefix), std.length(k) - std.length(prefix)); + { [target] +: { [suffix]:: v } }, + + local routes = { + "cassandra-": route("cassandra"), + "pulsar-": route("pulsar"), + "qdrant-": route("qdrant"), + "api-gateway-": route("api-gateway"), + "librarian-": route("librarian"), + }, + + with_params:: function(pars) + std.foldl( + function(acc, k) + local matchingPrefixes = [p for p in std.objectFields(routes) if std.startsWith(k, p)]; + if std.length(matchingPrefixes) > 0 then + local prefix = matchingPrefixes[0]; + acc + routes[prefix](prefix, k, pars[k]) + else + acc + { [k]:: pars[k] }, + std.objectFields(pars), + {} + ), + }, + + // Memory profiles + "memory-profile-low": import "profiles/memory-profile-low.jsonnet", + + // Model hosting + + "hosting-intel-battlemage-vllm": + import "model-hosting/intel-battlemage-vllm.jsonnet", + + "hosting-cpu-tgi": + import "model-hosting/cpu-tgi.jsonnet", + + "hosting-intel-xpu-tgi": + import "model-hosting/intel-xpu-tgi.jsonnet", + + "hosting-intel-gaudi-tgi": + import "model-hosting/intel-gaudi-tgi.jsonnet", + + "hosting-intel-xpu-vllm": + import "model-hosting/intel-xpu-vllm.jsonnet", + + "hosting-intel-gaudi-vllm": + import "model-hosting/intel-gaudi-vllm.jsonnet", + + "hosting-nvidia-gpu-vllm": + import "model-hosting/nvidia-gpu-vllm.jsonnet", + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/agent-manager-react.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/agent-manager-react.jsonnet new file mode 100644 index 00000000..954d80a7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/agent-manager-react.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "agent-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("agent-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "agent-manager-react", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "agent-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/chunker-recursive.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/chunker-recursive.jsonnet new file mode 100644 index 00000000..0b924aba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/chunker-recursive.jsonnet @@ -0,0 +1,55 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "chunk-size":: 2000, + "chunk-overlap":: 100, + + "chunker" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + std.toString($["chunk-size"]), + "--chunk-overlap", + std.toString($["chunk-overlap"]), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/configuration.jsonnet new file mode 100644 index 00000000..75afebb9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/configuration.jsonnet @@ -0,0 +1,57 @@ + +// This puts the default configuration together. References many things, +// flow classes, a default flow, token costs, prompts, agent tools + +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "init-trustgraph" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local cfgVol = engine.configVolume( + "trustgraph-cfg", "trustgraph", + { + "config.json": importstr "trustgraph/config.json", + } + ); + + local container = + engine.container("init-trustgraph") + .with_image(images.trustgraph_flow) + .with_command( + [ + "tg-init-trustgraph", + "-p", + url.pulsar_admin, + "--config-file", + "/trustgraph/config.json", + ] + ) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_volume_mount(cfgVol, "/trustgraph/"); + + local containerSet = engine.containers( + "init-trustgraph", [ container ] + ); + + engine.resources([ + cfgVol, + containerSet, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/document-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/document-rag.jsonnet new file mode 100644 index 00000000..e466e3a4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/document-rag.jsonnet @@ -0,0 +1,92 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "document-rag" +: { + + "doc-limit":: 20, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local docLimit = self["doc-limit"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "document-rag", + "-p", + url.pulsar, + "--doc-limit", + std.toString(docLimit), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "document-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "document-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/graph-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/graph-rag.jsonnet new file mode 100644 index 00000000..06ee5b3b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/graph-rag.jsonnet @@ -0,0 +1,283 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "kg-extract-definitions" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-definitions") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-definitions", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-definitions", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-relationships" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-relationships") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-relationships", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-relationships", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-agent" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-agent") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-agent", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-agent", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-ontology" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "300M", + "memory-reservation":: "300M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-ontology") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-ontology", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-ontology", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-rag" +: { + + concurrency:: 1, + "entity-limit":: 50, + "triple-limit":: 30, + "max-subgraph-size":: 400, + "max-path-length":: 2, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local entityLimit = self["entity-limit"]; + local tripleLimit = self["triple-limit"]; + local maxSubgraphSize = self["max-subgraph-size"]; + local maxPathLength = self["max-path-length"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("graph-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-rag", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--entity-limit", + std.toString(entityLimit), + "--triple-limit", + std.toString(tripleLimit), + "--max-subgraph-size", + std.toString(maxSubgraphSize), + "--max-path-length", + std.toString(maxPathLength), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "graph-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/librarian.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/librarian.jsonnet new file mode 100644 index 00000000..2ecda5a8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/librarian.jsonnet @@ -0,0 +1,58 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local garage = import "backends/garage.jsonnet"; +local cassandra = import "backends/cassandra.jsonnet"; + +{ + + "librarian" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("librarian") + .with_image(images.trustgraph_flow) + .with_command([ + "librarian", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + "--object-store-endpoint", + url.object_store, + "--object-store-access-key", + $.garage["access-key"], + "--object-store-secret-key", + $.garage["secret-key"], + "--object-store-region", + $.garage.region, + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "librarian", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +// Garage and Cassandra are used by the Librarian +} + garage + cassandra + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/mcp-server.jsonnet new file mode 100644 index 00000000..9bf7e3bd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/mcp-server.jsonnet @@ -0,0 +1,54 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "mcp-server" +: { + + port:: 8000, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local port = self.port; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("mcp-server-secret") + .with_env_var("MCP_SERVER_SECRET", "mcp-server-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local container = + engine.container("mcp-server") + .with_image(images.trustgraph_mcp) + .with_command([ + "mcp-server", + "--port", + std.toString(port), + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/prompt-overrides.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/prompt-overrides.jsonnet new file mode 100644 index 00000000..852ec09d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/prompt-overrides.jsonnet @@ -0,0 +1,24 @@ +local default_prompts = import "prompts/default-prompts.jsonnet"; + +{ + + with:: function(key, value) + if (key == "system-template") then + self + { + prompts +:: { + "system-template": value, + } + } + else + self + { + prompts +:: { + templates +:: { + [key] +:: { + prompt: value + } + } + } + }, + +} + default_prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/prompt-template.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/prompt-template.jsonnet new file mode 100644 index 00000000..3cd79d59 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/prompt-template.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "prompt" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("prompt") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "prompt", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "prompt-rag" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("prompt-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--id", + "prompt-rag", + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "prompt-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/rev-gateway.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/rev-gateway.jsonnet new file mode 100644 index 00000000..fbd9f77a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/rev-gateway.jsonnet @@ -0,0 +1,62 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "rev-gateway" +: { + + // Invalid, but at least means the rev-gateway won't connect to anything + // it shouldn't. + token:: "INVALID_TOKEN", + uri:: "wss://127.0.0.1/api/v1/relay?token=" + self.token, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local uri = self.uri; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("rev-gateway-secret") + .with_env_var("REV_GATEWAY_SECRET", "rev-gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "rev-gateway", + "-p", + url.pulsar, + "--websocket-uri", + std.toString(uri), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/structured-data.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/structured-data.jsonnet new file mode 100644 index 00000000..52ea85da --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/structured-data.jsonnet @@ -0,0 +1,211 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "nlp-query" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("nlp-query") + .with_image(images.trustgraph_flow) + .with_command([ + "nlp-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "nlp-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-query" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("structured-query") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "structured-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-diag" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "96M", + "memory-reservation":: "96M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("structured-diag") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-diag", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "structured-diag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-rows" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-rows") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-rows", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-rows", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "row-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("row-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "row-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "row-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/trustgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/trustgraph.jsonnet new file mode 100644 index 00000000..3cf12d26 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/core/trustgraph.jsonnet @@ -0,0 +1,473 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +local config_initialiser = import "configuration.jsonnet"; +local config = import "../runtime-config/trustgraph-config.jsonnet"; +local librarian = import "librarian.jsonnet"; +local mcp_server = import "mcp-server.jsonnet"; +local workbench = import "../ui/workbench-ui.jsonnet"; +local graphrag = import "graph-rag.jsonnet"; +local documentrag = import "document-rag.jsonnet"; +local prompt_template = import "prompt-template.jsonnet"; +local agent_manager = import "agent-manager-react.jsonnet"; +local structured_data = import "structured-data.jsonnet"; +local ddg = import "mcp/ddg-mcp-server.jsonnet"; + +// Helper to create a routing function for a target object +local route = function(target) + function(prefix, k, v) + local suffix = std.substr(k, std.length(prefix), std.length(k) - std.length(prefix)); + { [target] +: { [suffix]:: v } }; + +// Parameter prefix -> target object routing table +local routes = { + "prompt-rag-": route("prompt-rag"), + "prompt-": route("prompt"), + "text-completion-rag-": route("text-completion-rag"), + "text-completion-": route("text-completion"), + "embeddings-": route("embeddings"), + "api-gateway-": route("api-gateway"), + "chunk-": route("chunker"), + "graph-rag-": route("graph-rag"), + "graph-embeddings-": route("graph-embeddings"), + "kg-extract-definitions-": route("kg-extract-definitions"), + "kg-extract-relationships-": route("kg-extract-relationships"), + "kg-extract-agent-": route("kg-extract-agent"), + "kg-extract-ontology-": route("kg-extract-ontology"), + "kg-extract-objects-": route("kg-extract-objects"), + "garage-": route("garage"), + "config-svc-": route("config-svc"), + "pdf-decoder-": route("pdf-decoder"), + "mcp-tool-": route("mcp-tool"), + "mcp-server-": route("mcp-server"), + "metering-rag-": route("metering-rag"), + "metering-": route("metering"), + "kg-store-": route("kg-store"), + "kg-manager-": route("kg-manager"), + "librarian-": route("librarian"), + "agent-manager-": route("agent-manager"), + "document-rag-": route("document-rag"), + "document-embeddings-": route("document-embeddings"), + "rev-gateway-": route("rev-gateway"), + "nlp-query-": route("nlp-query"), + "structured-query-": route("structured-query"), + "structured-diag-": route("structured-diag"), + "init-trustgraph-": route("init-trustgraph"), +}; + +// Find longest matching prefix (most specific first) +local findRoute = function(k) + local prefixes = std.objectFields(routes); + local matching = std.filter(function(p) std.startsWith(k, p), prefixes); + local sorted = std.sort(matching, function(x) -std.length(x)); + if std.length(sorted) > 0 then sorted[0] else null; + +{ + + // Route parameters to appropriate internal objects based on prefix + with:: function(k, v) + local prefix = findRoute(k); + if prefix != null then + self + routes[prefix](prefix, k, v) + else + self + { [k]:: v }, + + "log-level":: "INFO", + + // Base objects with concurrency defaults (LLM/embeddings components merge into these) + "text-completion" +: { concurrency:: 1 }, + "text-completion-rag" +: { concurrency:: 1 }, + embeddings +: { concurrency:: 1 }, + + "api-gateway" +: { + + port:: 8088, + timeout:: 600, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local port = self.port; + local timeout = self.timeout; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("gateway-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "api-gateway", + "-p", + url.pulsar, + "--timeout", + std.toString(timeout), + "--port", + std.toString(port), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "chunker" +: { + + size:: 2000, + overlap:: 50, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local size = self.size; + local overlap = self.overlap; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + std.toString(size), + "--chunk-overlap", + std.toString(overlap), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "config-svc" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local cpuLimit = self["cpu-limit"]; + local cpuReservation = self["cpu-reservation"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("config-svc") + .with_image(images.trustgraph_flow) + .with_command([ + "config-svc", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(cpuLimit, memoryLimit) + .with_reservations(cpuReservation, memoryReservation); + + local containerSet = engine.containers( + "config-svc", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "pdf-decoder" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("pdf-decoder") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-decoder", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "pdf-decoder", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "mcp-tool" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("mcp-tool") + .with_image(images.trustgraph_flow) + .with_command([ + "mcp-tool", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "mcp-tool", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("metering") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "metering", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering-rag" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("metering-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--id", + "metering-rag", + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "metering-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-store" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-store") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-store", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-store", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-manager", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + librarian + mcp_server + workbench + graphrag + + documentrag + prompt_template + agent_manager + structured_data + + config_initialiser + config + + ddg + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/embeddings/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/embeddings/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..37116de0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/embeddings/embeddings-fastembed.jsonnet @@ -0,0 +1,49 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-fastembed.jsonnet"; + +{ + + "fastembed-models":: models, + + "embeddings-models" +:: $["fastembed-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-fastembed", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/embeddings/embeddings-hf.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/embeddings/embeddings-hf.jsonnet new file mode 100644 index 00000000..492f05cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/embeddings/embeddings-hf.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-huggingface.jsonnet"; + +{ + + "huggingface-embeddings-models":: models, + + "embeddings-models" +:: $["huggingface-embeddings-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_hf) + .with_command([ + "embeddings-hf", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/embeddings/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/embeddings/embeddings-ollama.jsonnet new file mode 100644 index 00000000..05cbed72 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/embeddings/embeddings-ollama.jsonnet @@ -0,0 +1,52 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local models = import "parameters/embeddings-ollama.jsonnet"; + +{ + + "ollama-url":: "${OLLAMA_HOST}", + + "ollama-models":: models, + + "embeddings-models" +:: $["ollama-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-r", + $["ollama-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/aks-k8s.jsonnet new file mode 100644 index 00000000..07e050f2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/aks-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "disk.csi.azure.com", + parameters: { + // Standard disks (spinning magnetic), Locally Redundant Storage + // Cheapest, basically + skuName: "Standard_LRS", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/docker-compose.jsonnet new file mode 100644 index 00000000..334b3517 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/docker-compose.jsonnet @@ -0,0 +1,260 @@ +{ + + // Extract resources using the engine + package:: function(patterns) + std.foldl( + function(state, p) state + p.create(self), + std.objectValues(patterns), + {} + ), + + container:: function(name) + { + + local container = self, + + name:: name, + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_group:: function(x) self + + if std.objectHas(container, "group_add") then + { group_add: container.group_add + [x] } + else + { group_add: [x] }, + + with_command:: function(x) self + { + command: + if std.isString(x) then + std.strReplace(x, "$", "$$") + else if std.isArray(x) then + std.map(function(s) std.strReplace(s, "$", "$$"), x) + else + x + }, + + with_entrypoint:: function(x) self + { entrypoint: x }, + + with_runtime:: function(x) self + { runtime: x }, + + with_privileged:: function(x) self + { privileged: x }, + + with_ipc:: function(x) self + { ipc: x }, + + with_capability:: function(x) self + + if std.objectHas(container, "capability") then + { cap_add: container.capability + x } + else + { cap_add: [x], }, + + with_environment:: function(x) self + + if std.objectHas(container, "environment") then + { environment: container.environment + x } + else + { environment: x, }, + + with_device:: function(hdev, cdev) self + + if std.objectHas(container, "devices") then + { devices: container.devices + [ "%s:%s" % [hdev, cdev] ] } + else + { devices: [ "%s:%s" % [hdev, cdev] ], }, + + with_limits:: function(c, m) self + { + deploy +: { resources +: { + limits: { cpus: c, memory: m } + } }, + }, + + with_reservations:: function(c, m) self + { + deploy +: { resources +: { + reservations: { cpus: c, memory: m } + } }, + }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [vol.volid, mnt] + ] + else + [ + "%s:%s" % [vol.volid, mnt] + ] + }, + + with_bind_mount:: + function(src, dest) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [src, dest] + ] + else + [ + "%s:%s" % [src, dest] + ] + }, + + with_port:: + function(src, dest, name) + self + { + ports: + if std.objectHas(container, "ports") then + container.ports + [ "%d:%d" % [src, dest] ] + else + [ "%d:%d" % [src, dest] ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj.with_environment( + { [x]: "${" + x + "}" } + ), + vars.variables, + self + ), + + restart: "on-failure:100", + + add:: function() { + services +: { + [container.name]: container, + } + } + + }, + + internalService:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + with_size:: function(size) self + { size: size }, + + add:: function() { + volumes +: { + [volume.name]: {} + } + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: "./" + dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + variables:: [], + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + }, + + add:: function() { + } + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.foldl( + function(state, c) state + c.add(), + cont.containers, + {} + ), + + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/eks-k8s.jsonnet new file mode 100644 index 00000000..3fc3f035 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/eks-k8s.jsonnet @@ -0,0 +1,46 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "ebs.csi.aws.com", + parameters: { + type: "gp3", + encrypted: "true", + iops: "6000", + throughput: "400", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/gcp-k8s.jsonnet new file mode 100644 index 00000000..71792426 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/gcp-k8s.jsonnet @@ -0,0 +1,44 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "pd.csi.storage.gke.io", + parameters: { + type: "pd-balanced", + "csi.storage.k8s.io/fstype": "ext4", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/k8s.jsonnet new file mode 100644 index 00000000..2067f6ac --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/k8s.jsonnet @@ -0,0 +1,393 @@ +{ + + container:: function(name) + { + + local container = self, + + name: name, + limits: {}, + reservations: {}, + ports: [], + volumes: [], + bindMounts: [], + groups: [], + environment: [], + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_group:: function(x) self + { groups: super.groups + [x] }, + + with_privileged:: function(x) self + { privileged: x }, + + with_command:: function(x) self + { command: x }, + + with_entrypoint:: function(x) self + { entrypoint: x }, + + with_environment:: function(x) self + { + environment: super.environment + [ + { + name: v.key, value: v.value + } + for v in std.objectKeysValues(x) + ], + }, + + with_limits:: function(c, m) self + { limits: { cpu: c, memory: m } }, + + with_reservations:: + function(c, m) self + { reservations: { cpu: c, memory: m } }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: super.volumes + [{ + volume: vol, mount: mnt + }] + }, + + with_bind_mount:: + function(src, dest) + local name = "bind-" + std.strReplace(std.strReplace(src, "/", "-"), ".", "-"); + self + { + bindMounts: super.bindMounts + [{ + name: name, src: src, dest: dest + }] + }, + + with_port:: + function(src, dest, name) self + { + ports: super.ports + [ + { src: src, dest: dest, name : name } + ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj + { + environment: super.environment + [{ + name: x, + valueFrom: { + secretKeyRef: { + name: vars.name, + key: vars.keyMap[x], + } + } + }] + }, + vars.variables, + self + ), + + add:: function() [ + + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: container.name, + namespace: "trustgraph", + labels: { + app: container.name + } + }, + spec: { + replicas: 1, + selector: { + matchLabels: { + app: container.name, + } + }, + template: { + metadata: { + labels: { + app: container.name, + } + }, + spec: { + containers: [ + { + name: container.name, + image: container.image, + + // FIXME: Make everything run as + // root. Needed to get filesystems + // to be accessible. There's a + // better way of doing this? + securityContext: { + runAsUser: 0, + runAsGroup: 0, + } + ( + if std.objectHas(container, "privileged") && container.privileged then + { privileged: true } + else {} + ), + + resources: { + requests: container.reservations, + limits: container.limits + }, + } + ( + if std.length(container.ports) > 0 then + { + ports: [ + { + hostPort: port.src, + containerPort: port.dest, + } + for port in container.ports + ] + } else + {}) + + + (if std.objectHas(container, "entrypoint") then + // Entrypoint is set - use command for entrypoint, args for command + (if std.isString(container.entrypoint) && container.entrypoint == "" then + { command: [] } + else if std.isArray(container.entrypoint) then + { command: container.entrypoint } + else + { command: [container.entrypoint] } + ) + (if std.objectHas(container, "command") then + { args: container.command } + else {}) + else if std.objectHas(container, "command") then + { command: container.command } + else {}) + + + (if std.length(container.environment) > 0 then + { + env: container.environment, + } + else {}) + + + (if std.length(container.volumes) > 0 || std.length(container.bindMounts) > 0 then + { + volumeMounts: [ + { + mountPath: vol.mount, + name: vol.volume.name, + } + for vol in container.volumes + ] + [ + { + mountPath: bm.dest, + name: bm.name, + } + for bm in container.bindMounts + ] + } + + else + {} + ) + ], + volumes: [ + vol.volume.volRef() + for vol in container.volumes + ] + [ + { + name: bm.name, + hostPath: { path: bm.src } + } + for bm in container.bindMounts + ] + } + ( + if std.length(container.groups) > 0 then + { securityContext: { supplementalGroups: container.groups } } + else {} + ) + }, + } + {} + + } + + ] + + }, + + // Just an alias + internalService:: self.service, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + ports: [], + + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + + add:: function() [ + + { + + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + storageClassName: "tg", + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + data: parts + }, + ], + + + volRef:: function() { + name: volume.name, + configMap: { name: volume.name }, + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + variables: [], + keyMap: {}, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + }, + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + keyMap: super.keyMap + { [name]: key }, + }, + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.flattenArrays( + [ c.add() for c in cont.containers ] + ), + + }, + + resources:: function(res) + + std.flattenArrays( + [ c.add() for c in res ] + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/minikube-k8s.jsonnet new file mode 100644 index 00000000..858b17ad --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/minikube-k8s.jsonnet @@ -0,0 +1,115 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList, + + volume:: function(name) + { + local volume = self, + name: name, + with_size:: function(size) self + { size: size }, + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolume", + metadata: { + name: volume.name, + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + capacity: { + storage: volume.size, + }, + persistentVolumeReclaimPolicy: "Delete", + hostPath: { + path: "/data/pv-" + volume.name, + }, + } + }, + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + service:: function(containers) + { + local service = self, + name: containers.name, + ports: [], + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + add:: function() [ + { + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + type: "LoadBalancer", + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/noop.jsonnet new file mode 100644 index 00000000..1f384648 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/noop.jsonnet @@ -0,0 +1,79 @@ +{ + + // Extract resources usnig the engine + package:: function(patterns) {}, + + container:: function(name) { + + with_image:: function(x) self + {}, + + with_user:: function(x) self + {}, + + with_command:: function(x) self + {}, + + with_runtime:: function(x) self + {}, + + with_privileged:: function(x) self + {}, + + with_ipc:: function(x) self + {}, + + with_capability:: function(x) self + {}, + + with_environment:: function(x) self + {}, + + with_device:: function(hdev, cdev) self + {}, + + with_limits:: function(c, m) self + {}, + + with_reservations:: function(c, m) self + {}, + + with_volume_mount:: self + {}, + + with_port:: function(src, dest, name) self + {}, + + with_env_var_secrets:: function(vars) self + {}, + + add:: function() {}, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + service:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + volume:: function(name) { + with_size:: function(size) self + {}, + add:: function() {}, + }, + + configVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + secretVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self + {}, + add:: function() {}, + }, + + containers:: function(name, containers) { + add:: function() {}, + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/ovh-k8s.jsonnet new file mode 100644 index 00000000..15d57c83 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/ovh-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "cinder.csi.openstack.org", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", + parameters: { + availability: "nova", + fsType: "ext4", + type: "high-speed", + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/scw-k8s.jsonnet new file mode 100644 index 00000000..eed23968 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/engine/scw-k8s.jsonnet @@ -0,0 +1,40 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "csi.scaleway.com", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/agent-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/agent-extract.jsonnet new file mode 100644 index 00000000..4c5785c8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/agent-extract.jsonnet @@ -0,0 +1,35 @@ +// Agent-based extraction module +// Uses AI agents for more sophisticated knowledge extraction from text +// Leverages agent tools and reasoning for complex extraction tasks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - internal agent extraction service + "interfaces" +: { + }, + + // No configurable parameters for agent extraction + "parameters" +: { + }, + + // Flow-level processors for agent-based extraction + "flow" +: { + // Agent-based knowledge extraction processor + // Uses AI agents with tools to extract structured knowledge + "kg-extract-agent:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output knowledge triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "agent-request": request("agent:{id}"), // Agent service requests + "agent-response": response("agent:{id}"), // Agent service responses + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/agent.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/agent.jsonnet new file mode 100644 index 00000000..a483b21d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/agent.jsonnet @@ -0,0 +1,56 @@ +// Agent management module +// Provides AI agent orchestration and tool integration +// Manages agent conversations, tool calls, and response coordination +// Supports MCP tools, GraphRAG, and structured queries + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services (agent requires LLM for reasoning, MCP for tools) +local llm_services = import "llm-services.jsonnet"; +local mcp_service = import "mcp-service.jsonnet"; + +// Merge shared services with agent-specific configuration +llm_services + mcp_service + { + + // External interfaces for agent operations + "interfaces" +: { + "agent": request_response("agent:{id}"), + }, + + // Flow-level processors for agent management + "flow" +: { + // Agent manager orchestrates agent conversations and tool usage + "agent-manager:{id}": { + + // Agent communication channels + request: request("agent:{id}"), + next: request("agent:{id}"), + response: response("agent:{id}"), + + // LLM and prompt services + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + + // Tool integrations + "mcp-tool-request": request("mcp-tool:{id}"), + "mcp-tool-response": response("mcp-tool:{id}"), + "graph-rag-request": request("graph-rag:{id}"), + "graph-rag-response": response("graph-rag:{id}"), + "structured-query-request": request("structured-query:{id}"), + "structured-query-response": response("structured-query:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "row-embeddings-query-request": request("row-embeddings:{id}"), + "row-embeddings-query-response": response("row-embeddings:{id}"), + }, + }, + + // Blueprint-level processors for agent-related services + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/document-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/document-store.jsonnet new file mode 100644 index 00000000..16f304fa --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/document-store.jsonnet @@ -0,0 +1,56 @@ +// Document store module +// Infrastructure for document-based RAG using chunk embeddings +// Handles document embedding storage, retrieval, and question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with document store configuration +llm_services + embeddings_service + { + + // External interfaces for document store + "interfaces" +: { + // Document embedding storage and retrieval + "document-embeddings-store": flow("document-embeddings-store:{id}"), + "document-rag": request_response("document-rag:{id}"), + "document-embeddings": request_response("document-embeddings:{id}"), + }, + + // Flow-level processors for document embedding and storage + "flow" +: { + "document-embeddings:{id}": { + input: flow("chunk-load:{id}"), + output: flow("document-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "doc-embeddings-write:{id}": { + input: flow("document-embeddings-store:{id}"), + }, + "document-rag:{id}": { + request: request("document-rag:{id}"), + response: response("document-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "document-embeddings-request": request("document-embeddings:{id}"), + "document-embeddings-response": response("document-embeddings:{id}"), + }, + "doc-embeddings-query:{id}": { + request: request("document-embeddings:{id}"), + response: response("document-embeddings:{id}"), + }, + }, + + // Blueprint-level processors for document RAG operations + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/embeddings-service.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/embeddings-service.jsonnet new file mode 100644 index 00000000..1537e6c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/embeddings-service.jsonnet @@ -0,0 +1,30 @@ +// Shared embeddings service module +// Provides vector embedding generation for text +// Import this module in any flow that requires embeddings + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // Interfaces exposed by embeddings service + "interfaces" +: { + "embeddings": request_response("embeddings:{id}"), + }, + + "parameters" +: { + }, + + // Flow-level processor for embeddings + "flow" +: { + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/flow-blueprints.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/flow-blueprints.jsonnet new file mode 100644 index 00000000..4282fa23 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/flow-blueprints.jsonnet @@ -0,0 +1,48 @@ +// TrustGraph Flow Blueprints Configuration +// +// RAG Modes (4 types): +// - Document RAG: Uses document chunk embeddings +// - Graph RAG: Extracts definitions + relationships to graph +// - Ontology RAG: Extracts using ontology definitions to graph (mutually exclusive with Graph RAG) +// - Structured RAG: Extracts objects to object store +// +// Module structure: +// - *-store: Storage and query infrastructure +// - *-extract: Extraction methods + +// Import all the modular flow components +local graph_store = import "graph-store.jsonnet"; +local document_store = import "document-store.jsonnet"; +local structured_store = import "structured-store.jsonnet"; +local graphrag_extract = import "graphrag-extract.jsonnet"; +local ontorag_extract = import "ontorag-extract.jsonnet"; +local structured_extract = import "structured-extract.jsonnet"; +local agent = import "agent.jsonnet"; +local load = import "load.jsonnet"; +local kgcore = import "kgcore.jsonnet"; + +{ + + // Full system: Graph RAG + Document RAG + knowledge cores + "everything": { + description: "Graph RAG + Document RAG + knowledge cores", + tags: ["document-rag", "graph-rag", "kgcore"], + } + + graph_store + document_store + agent + load + + graphrag_extract + kgcore, + + // Structured RAG only + "structured": { + description: "Structured data extraction and querying", + tags: ["structured"], + } + + structured_store + structured_extract + agent + load, + + // Ontology RAG + knowledge cores + "ontology": { + description: "Ontology RAG + knowledge cores", + tags: ["onto-rag", "kgcore"], + } + + graph_store + ontorag_extract + agent + load + kgcore, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/graph-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/graph-store.jsonnet new file mode 100644 index 00000000..9bd49b9c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/graph-store.jsonnet @@ -0,0 +1,70 @@ +// Graph store module +// Shared infrastructure for graph-based RAG (used by both GraphRAG and OntologyRAG) +// Handles knowledge graph storage, embeddings, and graph-based question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with graph store configuration +llm_services + embeddings_service + { + + // External interfaces exposed by the graph store + "interfaces" +: { + // Data ingestion interfaces for graph construction + "entity-contexts-load": flow("entity-contexts-load:{id}"), + "triples-store": flow("triples-store:{id}"), + "graph-embeddings-store": flow("graph-embeddings-store:{id}"), + + // Query interfaces for graph-based operations + "graph-rag": request_response("graph-rag:{id}"), + "triples": request_response("triples:{id}"), + "graph-embeddings": request_response("graph-embeddings:{id}"), + }, + + // Flow-level processors - handle data streams for a specific flow instance + "flow" +: { + "graph-embeddings:{id}": { + input: flow("entity-contexts-load:{id}"), + output: flow("graph-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "triples-write:{id}": { + input: flow("triples-store:{id}"), + }, + "graph-embeddings-write:{id}": { + input: flow("graph-embeddings-store:{id}"), + }, + "graph-rag:{id}": { + request: request("graph-rag:{id}"), + response: response("graph-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "graph-embeddings-request": request("graph-embeddings:{id}"), + "graph-embeddings-response": response("graph-embeddings:{id}"), + "triples-request": request("triples:{id}"), + "triples-response": response("triples:{id}"), + }, + "triples-query:{id}": { + request: request("triples:{id}"), + response: response("triples:{id}"), + }, + "graph-embeddings-query:{id}": { + request: request("graph-embeddings:{id}"), + response: response("graph-embeddings:{id}"), + }, + }, + + // Blueprint-level processors - shared across all flow instances of this blueprint + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/graphrag-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/graphrag-extract.jsonnet new file mode 100644 index 00000000..6bcbca14 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/graphrag-extract.jsonnet @@ -0,0 +1,44 @@ +// GraphRAG extraction module +// Extraction method for GraphRAG - extracts definitions and relationships +// Mutually exclusive with OntologyRAG extraction (both write to graph store) + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts entity definitions from text chunks + // Identifies and defines key entities mentioned in the text + "kg-extract-definitions:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output definition triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition extraction prompts + "prompt-response": response("prompt:{id}"), + }, + + // Extracts relationships between entities + // Identifies how entities are connected and interact + "kg-extract-relationships:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output relationship triples + "prompt-request": request("prompt:{id}"), // Relationship extraction prompts + "prompt-response": response("prompt:{id}"), + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/helpers.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/helpers.jsonnet new file mode 100644 index 00000000..eabb3bf4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/helpers.jsonnet @@ -0,0 +1,29 @@ +// Helper functions for flow configuration +// Provides utility functions for constructing flow, request, and response URIs +// used throughout the TrustGraph flow configuration system + +// Creates a persistent flow URI for data streams +// Persistent flows retain messages until consumed +local flow(x) = "persistent://tg/flow/" + x; + +// Creates a non-persistent request URI for request-response patterns +// Non-persistent means messages are not retained if no consumer is present +local request(x) = "non-persistent://tg/request/" + x; + +// Creates a non-persistent response URI for request-response patterns +local response(x) = "non-persistent://tg/response/" + x; + +// Creates a request-response pair for bidirectional communication +// Returns an object with both request and response URIs +local request_response(x) = { + request: request(x), + response: response(x), +}; + +// Export all helper functions for use in other modules +{ + flow: flow, + request: request, + response: response, + request_response: request_response, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/kgcore.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/kgcore.jsonnet new file mode 100644 index 00000000..bfc01ada --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/kgcore.jsonnet @@ -0,0 +1,31 @@ +// Knowledge Graph Core storage module +// Handles persistent storage of knowledge graph data +// Consolidates triples and graph embeddings into permanent storage +// Creates the core knowledge base for long-term use + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; + +{ + // No external interfaces - internal storage service + "interfaces" +: { + }, + + // No configurable parameters for core storage + "parameters" +: { + }, + + // Flow-level processors for knowledge graph storage + "flow" +: { + // Knowledge graph store consolidates extracted knowledge + // Takes processed triples and embeddings and stores them permanently + "kg-store:{id}": { + "triples-input": flow("triples-store:{id}"), // Input RDF triples stream + "graph-embeddings-input": flow("graph-embeddings-store:{id}"), // Input graph embeddings + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/llm-parameters.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/llm-parameters.jsonnet new file mode 100644 index 00000000..a99f72ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/llm-parameters.jsonnet @@ -0,0 +1,59 @@ +{ + + // LLM model selection for normal LLM + "llm-model": { + "type": "llm-model", + "description": "LLM model", + "order": 1, + "advanced": false, + }, + + // LLM model for RAG operations + "llm-rag-model": { + "type": "llm-model", + "description": "LLM model for RAG", + "order": 2, + "advanced": true, + "controlled-by": "llm-model", + }, + + // LLM model selection for normal LLM + "llm-temperature": { + "type": "llm-temperature", + "description": "LLM temperature", + "order": 3, + "advanced": true, + }, + + // LLM model selection for normal LLM + "llm-rag-temperature": { + "type": "llm-temperature", + "description": "LLM temperature for RAG", + "order": 4, + "advanced": true, + }, + + "embeddings-model": { + "type": "embeddings-model", + "description": "Embeddings model", + "order": 5, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-size": { + "type": "chunk-size", + "description": "Chunk size", + "order": 6, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-overlap": { + "type": "chunk-overlap", + "description": "Chunk overlap", + "order": 7, + "advanced": true, + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/llm-services.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/llm-services.jsonnet new file mode 100644 index 00000000..ee6e09ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/llm-services.jsonnet @@ -0,0 +1,66 @@ +// Shared LLM services module +// Provides text completion, prompt processing, and metering services +// Import this module in any flow that requires LLM functionality + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // Interfaces exposed by LLM services + "interfaces" +: { + "prompt": request_response("prompt:{id}"), + "text-completion": request_response("text-completion:{id}"), + }, + + // LLM configuration parameters + "parameters" +: llm_parameters, + + // Flow-level processors for LLM services + "flow" +: { + // Primary text completion service + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + + // RAG-specific text completion (may use different model) + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + + // Prompt processing service + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + + // RAG-specific prompt processing + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + + // Usage metering for primary completion + "metering:{id}": { + input: response("text-completion:{id}"), + }, + + // Usage metering for RAG completion + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/load.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/load.jsonnet new file mode 100644 index 00000000..6c440792 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/load.jsonnet @@ -0,0 +1,43 @@ +// Document loading and preprocessing module +// Handles document ingestion, format conversion, and chunking +// Converts PDFs to text and splits documents into processable chunks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services (load requires embeddings for chunk processing) +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with load-specific configuration +embeddings_service + { + + // External interfaces for document loading + "interfaces" +: { + "document-load": flow("document-load:{id}"), + "text-load": flow("text-document-load:{id}"), + }, + + // Flow-level processors for document preprocessing + "flow" +: { + // PDF decoder converts PDF documents to text + "pdf-decoder:{id}": { + input: flow("document-load:{id}"), + output: flow("text-document-load:{id}"), + }, + + // Chunker splits documents into smaller, processable pieces + "chunker:{id}": { + input: flow("text-document-load:{id}"), + output: flow("chunk-load:{id}"), + "chunk-size": "{chunk-size}", + "chunk-overlap": "{chunk-overlap}", + }, + }, + + // Blueprint-level processors for document loading services + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/mcp-service.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/mcp-service.jsonnet new file mode 100644 index 00000000..5d599a67 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/mcp-service.jsonnet @@ -0,0 +1,31 @@ +// Shared MCP (Model Context Protocol) tool service module +// Provides MCP tool execution capabilities for agents +// Import this module in any flow that requires MCP tool integration + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // Interfaces exposed by MCP service + "interfaces" +: { + "mcp-tool": request_response("mcp-tool:{id}"), + }, + + "parameters" +: { + }, + + // Flow-level processor for MCP tool execution + "flow" +: { + "mcp-tool:{id}": { + request: request("mcp-tool:{id}"), + response: response("mcp-tool:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/ontorag-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/ontorag-extract.jsonnet new file mode 100644 index 00000000..e5521d8a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/ontorag-extract.jsonnet @@ -0,0 +1,39 @@ +// OntologyRAG extraction module +// Extraction method for OntologyRAG - extracts using ontology definitions +// Mutually exclusive with GraphRAG extraction (both write to graph store) + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts using ontology definitions + "kg-extract-ontology:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition + // extraction prompts + "prompt-response": response("prompt:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/structured-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/structured-extract.jsonnet new file mode 100644 index 00000000..54f6a05d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/structured-extract.jsonnet @@ -0,0 +1,30 @@ +// Structured RAG extraction module +// Extracts structured rows from text chunks +// Outputs to rows-store for structured data querying + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + "interfaces" +: { + }, + + "parameters" +: { + }, + + // Flow-level processor for structured row extraction + "flow" +: { + "kg-extract-rows:{id}": { + input: flow("chunk-load:{id}"), + output: flow("rows-store:{id}"), + "entity-contexts": flow("entity-contexts-load:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/structured-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/structured-store.jsonnet new file mode 100644 index 00000000..3668222d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/flows/structured-store.jsonnet @@ -0,0 +1,78 @@ +// Structured store module +// Shared infrastructure for structured data RAG +// Handles row storage, retrieval, and NLP query capabilities + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with structured store configuration +llm_services + embeddings_service + { + + // External interfaces for structured store + "interfaces" +: { + // Row storage and querying + "rows-store": flow("rows-store:{id}"), + "row-embeddings-store": flow("row-embeddings-store:{id}"), + "rows": request_response("rows:{id}"), + "row-embeddings": request_response("row-embeddings:{id}"), + + // Query interfaces + "nlp-query": request_response("nlp-query:{id}"), + "structured-query": request_response("structured-query:{id}"), + "structured-diag": request_response("structured-diag:{id}"), + }, + + // Flow-level processors for structured storage and query + "flow" +: { + "row-embeddings:{id}": { + input: flow("rows-store:{id}"), + output: flow("row-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "rows-write:{id}": { + input: flow("rows-store:{id}"), + }, + "row-embeddings-write:{id}": { + input: flow("row-embeddings-store:{id}"), + }, + "rows-query:{id}": { + request: request("rows:{id}"), + response: response("rows:{id}"), + }, + "row-embeddings-query:{id}": { + request: request("row-embeddings:{id}"), + response: response("row-embeddings:{id}"), + }, + "nlp-query:{id}": { + request: request("nlp-query:{id}"), + response: response("nlp-query:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + }, + "structured-query:{id}": { + request: request("structured-query:{id}"), + response: response("structured-query:{id}"), + "nlp-query-request": request("nlp-query:{id}"), + "nlp-query-response": response("nlp-query:{id}"), + "rows-query-request": request("rows:{id}"), + "rows-query-response": response("rows:{id}"), + }, + "structured-diag:{id}": { + request: request("structured-diag:{id}"), + response: response("structured-diag:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/azure-openai.jsonnet new file mode 100644 index 00000000..bee0fe47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/azure-openai.jsonnet @@ -0,0 +1,112 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure-openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-openai-" + key]:: value, + }, + +// Strategy is to specify the model with the AZURE_MODEL environment +// variable. This isn't something that can just be specified dynamically, +// it has to match what was provisioned in Azure. + + "azure-openai-max-output-tokens":: 4192, + "azure-openai-temperature":: 0.0, + "azure-openai-models":: models, + + "llm-models" +:: $["azure-openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/azure.jsonnet new file mode 100644 index 00000000..f4db7500 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/azure.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-" + key]:: value, + }, + + "azure-max-output-tokens":: 4096, + "azure-temperature":: 0.0, + "azure-models":: models, + + "llm-models" +:: $["azure-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/bedrock.jsonnet new file mode 100644 index 00000000..0db1475b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/bedrock.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/bedrock.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["bedrock-" + key]:: value, + }, + + "bedrock-max-output-tokens":: 4096, + "bedrock-temperature":: 0.0, + "bedrock-models":: models, + + "llm-models" +:: $["bedrock-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/claude.jsonnet new file mode 100644 index 00000000..f3125e96 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/claude.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/claude.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["claude-" + key]:: value, + }, + + "claude-max-output-tokens":: 4096, + "claude-temperature":: 0.0, + "claude-models":: models, + + "llm-models" +:: $["claude-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/cohere.jsonnet new file mode 100644 index 00000000..6517f7a0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/cohere.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/cohere.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["cohere-" + key]:: value, + }, + + "cohere-temperature":: 0.0, + "cohere-models":: models, + + "llm-models" +:: $["cohere-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/googleaistudio.jsonnet new file mode 100644 index 00000000..05a8592c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/googleaistudio.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/googleaistudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["googleaistudio-" + key]:: value, + }, + + "googleaistudio-max-output-tokens":: 4096, + "googleaistudio-temperature":: 0.0, + "googleaistudio-models":: models, + + "llm-models" +:: $["googleaistudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/llamafile.jsonnet new file mode 100644 index 00000000..f3bee2cc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/llamafile.jsonnet @@ -0,0 +1,94 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/slm.jsonnet"; +local models = import "parameters/llamafile.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["llamafile-" + key]:: value, + }, + + "llamafile-models":: models, + + "llm-models" +:: $["llamafile-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/lmstudio.jsonnet new file mode 100644 index 00000000..4cf78caa --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/lmstudio.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/lmstudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["lmstudio-" + key]:: value, + }, + + "lmstudio-max-output-tokens":: 4096, + "lmstudio-temperature":: 0.0, + "lmstudio-models":: models, + + "llm-models" +:: $["lmstudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/mistral.jsonnet new file mode 100644 index 00000000..2fb8c562 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/mistral.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/mistral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "mistral-max-output-tokens":: 4096, + "mistral-temperature":: 0.0, + "mistral-models":: models, + + "llm-models" +:: $["mistral-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/ollama.jsonnet new file mode 100644 index 00000000..6143f5cc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/ollama.jsonnet @@ -0,0 +1,102 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/ollama.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["ollama-" + key]:: value, + }, + + "ollama-models":: models, + + "llm-models" +:: $["ollama-models"], + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/openai.jsonnet new file mode 100644 index 00000000..c3d482fd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/openai.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["openai-" + key]:: value, + }, + + "openai-max-output-tokens":: 4096, + "openai-temperature":: 0.0, + "openai-models":: models, + + "llm-models" +:: $["openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/tgi.jsonnet new file mode 100644 index 00000000..91588dff --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/tgi.jsonnet @@ -0,0 +1,108 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-" + key]:: value, + }, + + "tgi-max-output-tokens":: 1024, + "tgi-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/vertexai.jsonnet new file mode 100644 index 00000000..cb8c141f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/vertexai.jsonnet @@ -0,0 +1,120 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vertexai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vertexai-" + key]:: value, + }, + + "vertexai-private-key":: "/vertexai/private.json", + "vertexai-region":: "us-central1", + "vertexai-max-output-tokens":: 4096, + "vertexai-temperature":: 0.0, + "vertexai-models":: models, + + "llm-models" +:: $["vertexai-models"], + + "text-completion" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion-rag") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/vllm.jsonnet new file mode 100644 index 00000000..8b846bbf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/llm/vllm.jsonnet @@ -0,0 +1,113 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vllm.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-" + key]:: value, + }, + + "vllm-models":: models, + + "llm-models" +:: $["vllm-models"], + + "vllm-max-output-tokens":: 1024, + "vllm-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/mcp/ddg-mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/mcp/ddg-mcp-server.jsonnet new file mode 100644 index 00000000..04176f15 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/mcp/ddg-mcp-server.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "ddg-mcp-server-port":: 9870, + + "ddg-mcp-server" +: { + + create:: function(engine) + + local port = $["ddg-mcp-server-port"]; + + local container = + engine.container("ddg-mcp-server") + .with_image(images["ddg-mcp-server"]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "ddg-mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + mcp +:: { + "duckduckgo": { + "remote-name": "search", + local port = $["ddg-mcp-server-port"], + local url = "http://ddg-mcp-server:%s/mcp" % port, + "url": url, + } + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/cpu-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/cpu-tgi.jsonnet new file mode 100644 index 00000000..04b61566 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/cpu-tgi.jsonnet @@ -0,0 +1,68 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + "tgi-service-storage":: "20G", + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-cpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--cuda-graphs", + "0", + ]) + .with_environment({ + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-battlemage-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-battlemage-vllm.jsonnet new file mode 100644 index 00000000..7646398c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-battlemage-vllm.jsonnet @@ -0,0 +1,98 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-Nemo-Instruct-2407", + "vllm-service-cpus":: "32.0", + "vllm-service-memory":: "48G", + "vllm-service-storage":: "48G", + "vllm-service-tokenizer-mode":: "mistral", + "vllm-service-datatype":: "float16", + "vllm-service-quantization":: "woq_int4", + "vllm-service-hf-token":: null, + "vllm-service-max-model-len":: 8192, + "vllm-service-max-num-seqs":: 16, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-battlemage"]) + .with_entrypoint("") // Clear default entrypoint + .with_command([ + "python", + "-m", + "ipex_llm.vllm.xpu.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--device", + "xpu", + "--dtype", + $["vllm-service-datatype"], + "--enforce-eager", + "--max-model-len", + std.toString($["vllm-service-max-model-len"]), + "--max-num-seqs", + std.toString($["vllm-service-max-num-seqs"]), + "--load-in-low-bit", + $["vllm-service-quantization"], + "--trust-remote-code", + "--tokenizer-mode", + $["vllm-service-tokenizer-mode"], + "--disable-sliding-window", + ]) + .with_environment({ + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS: "1", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-gaudi-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-gaudi-tgi.jsonnet new file mode 100644 index 00000000..5af36b78 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-gaudi-tgi.jsonnet @@ -0,0 +1,96 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "meta-llama/Llama-3.3-70B-Instruct", + "tgi-service-cpus":: "64.0", + "tgi-service-memory":: "64G", + "tgi-service-storage":: "50G", + "tgi-service-num-shard":: 8, + "tgi-service-max-input-tokens":: 4096, + "tgi-service-max-total-tokens":: 4096, + "tgi-service-max-batch-size":: 128, + "tgi-service-max-concurrent-requests":: 512, + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-gaudi"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--sharded", + "true", + "--num-shard", + std.toString($["tgi-service-num-shard"]), + "--max-input-tokens", + std.toString($["tgi-service-max-input-tokens"]), + "--max-total-tokens", + std.toString($["tgi-service-max-total-tokens"]), + "--max-batch-size", + std.toString($["tgi-service-max-batch-size"]), + "--max-waiting-tokens", + "7", + "--max-concurrent-requests", + std.toString($["tgi-service-max-concurrent-requests"]), + "--cuda-graphs", + "0", + ]) + .with_runtime("habana") + .with_environment({ + HABANA_VISIBLE_DEVICES: "all", + OMPI_MCA_btl_vader_single_copy_mechanism: "none", + ENABLE_HPU_GRAPH: "true", + LIMIT_HPU_GRAPH: "true", + USE_FLASH_ATTENTION: "true", + FLASH_ATTENTION_RECOMPUTE: "true", + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-gaudi-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-gaudi-vllm.jsonnet new file mode 100644 index 00000000..81d31c14 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-gaudi-vllm.jsonnet @@ -0,0 +1,76 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "64.0", + "vllm-service-memory":: "64G", + "vllm-service-storage":: "50G", + "vllm-service-tensor-parallel-size":: 8, + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-gaudi"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--tensor-parallel-size", + std.toString($["vllm-service-tensor-parallel-size"]), + ]) + .with_runtime("habana") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + HABANA_VISIBLE_DEVICES: "all", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-xpu-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-xpu-tgi.jsonnet new file mode 100644 index 00000000..bc806874 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-xpu-tgi.jsonnet @@ -0,0 +1,75 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + "tgi-service-storage":: "20G", + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-intel-xpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--cuda-graphs", + "0", + ]) + .with_environment({ + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_group("video") + .with_group("render") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-xpu-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-xpu-vllm.jsonnet new file mode 100644 index 00000000..f6e82905 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/intel-xpu-vllm.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "8.0", + "vllm-service-memory":: "16G", + "vllm-service-storage":: "20G", + "vllm-service-datatype":: "float16", + "vllm-service-max-model-len":: 4096, + "vllm-service-max-num-seqs":: 16, + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-xpu"]) + .with_command([ + "python", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--device", + "xpu", + "--dtype", + $["vllm-service-datatype"], + "--enforce-eager", + "--max-model-len", + std.toString($["vllm-service-max-model-len"]), + "--max-num-seqs", + std.toString($["vllm-service-max-num-seqs"]), + "--block-size", + "64", + "--gpu-memory-util", + "0.85", + "--trust-remote-code", + "--disable-sliding-window", + ]) + .with_environment({ + VLLM_USE_V1: "1", + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_group("video") + .with_group("render") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/nvidia-gpu-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/nvidia-gpu-vllm.jsonnet new file mode 100644 index 00000000..d04876c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/model-hosting/nvidia-gpu-vllm.jsonnet @@ -0,0 +1,72 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-7B-Instruct-v0.3", + "vllm-service-cpus":: "0.5", + "vllm-service-memory":: "1G", + "vllm-service-storage":: "50G", + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-nvidia"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + ]) + .with_runtime("nvidia") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/monitoring/grafana.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/monitoring/grafana.jsonnet new file mode 100644 index 00000000..dd0dddd7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/monitoring/grafana.jsonnet @@ -0,0 +1,124 @@ +local images = import "values/images.jsonnet"; +local loki = import "loki.jsonnet"; + +{ + + "prometheus" +: { + + create:: function(engine) + + local vol = engine.volume("prometheus-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "prometheus-cfg", "prometheus", + { + "prometheus.yml": importstr "prometheus/prometheus.yml", + } + ); + + local container = + engine.container("prometheus") + .with_image(images.prometheus) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M") + .with_port(9090, 9090, "http") + .with_volume_mount(cfgVol, "/etc/prometheus/") + .with_volume_mount(vol, "/prometheus"); + + local containerSet = engine.containers( + "prometheus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9090, 9090, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + + "grafana" +: { + + create:: function(engine) + + local vol = engine.volume("grafana-storage").with_size("20G"); + + local provDashVol = engine.configVolume( + "prov-dash", "grafana/provisioning/", + { + "dashboard.yml": + importstr "grafana/provisioning/dashboard.yml", + } + + ); + + local provDataVol = engine.configVolume( + "prov-data", "grafana/provisioning/", + { + "datasource.yml": + importstr "grafana/provisioning/datasource.yml", + } + + ); + + local dashVol = engine.configVolume( + "dashboards", "grafana/dashboards/", + { + "overview-dashboard.json": + importstr "grafana/dashboards/overview-dashboard.json", + "log-dashboard.json": + importstr "grafana/dashboards/log-dashboard.json", + } + + ); + + local container = + engine.container("grafana") + .with_image(images.grafana) + .with_environment({ + // GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin", + // GF_AUTH_ANONYMOUS_ENABLED: "true", + // GF_ORG_ROLE: "Admin", + GF_ORG_NAME: "trustgraph.ai", + // GF_SERVER_ROOT_URL: "https://example.com", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(3000, 3000, "cassandra") + .with_volume_mount(vol, "/var/lib/grafana") + .with_volume_mount( + provDashVol, "/etc/grafana/provisioning/dashboards/" + ) + .with_volume_mount( + provDataVol, "/etc/grafana/provisioning/datasources/" + ) + .with_volume_mount( + dashVol, "/var/lib/grafana/dashboards/" + ); + + local containerSet = engine.containers( + "grafana", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3000, 3000, "http"); + + engine.resources([ + vol, + provDashVol, + provDataVol, + dashVol, + containerSet, + service, + ]) + + }, + +} + loki + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/monitoring/loki.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/monitoring/loki.jsonnet new file mode 100644 index 00000000..0774f6cf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/monitoring/loki.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "loki" +: { + + create:: function(engine) + + local vol = engine.volume("loki-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "loki-cfg", "loki", + { + "local-config.yaml": importstr "loki/local-config.yaml", + } + ); + + local container = + engine.container("loki") + .with_image(images.loki) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(3100, 3100, "http") + .with_volume_mount(cfgVol, "/etc/loki/") + .with_volume_mount(vol, "/loki"); + + local containerSet = engine.containers( + "loki", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3100, 3100, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/ocr/mistral-ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/ocr/mistral-ocr.jsonnet new file mode 100644 index 00000000..f70e0cd6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/ocr/mistral-ocr.jsonnet @@ -0,0 +1,49 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "pdf-decoder" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("mistral-ocr") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-ocr-mistral", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "mistral-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/ocr/ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/ocr/ocr.jsonnet new file mode 100644 index 00000000..4f53aa5a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/ocr/ocr.jsonnet @@ -0,0 +1,37 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "pdf-decoder" +: { + + create:: function(engine) + + local container = + engine.container("pdf-ocr") + .with_image(images.trustgraph_ocr) + .with_command([ + "pdf-ocr", + "-p", + url.pulsar, + ]) + .with_limits("1.0", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "pdf-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/azure-openai.jsonnet new file mode 100644 index 00000000..dcf706d4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/azure-openai.jsonnet @@ -0,0 +1,9 @@ +// Azure OpenAI LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-5.2", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/azure.jsonnet new file mode 100644 index 00000000..6ffa5fdf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/azure.jsonnet @@ -0,0 +1,9 @@ +// Azure LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/bedrock.jsonnet new file mode 100644 index 00000000..6c2c7e90 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/bedrock.jsonnet @@ -0,0 +1,84 @@ +// AWS Bedrock LLM Model Definitions +// Defines available models and their configurations for AWS Bedrock + +{ + "type": "string", + "description": "LLM model to use", + "default": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + "enum": [ + // ── Anthropic Claude ────────────────────────────────────── + { + id: "global.anthropic.claude-opus-4-6-v1", + description: "Claude Opus 4.6 (maximum intelligence)" + }, + { + id: "global.anthropic.claude-opus-4-5-20251101-v1:0", + description: "Claude Opus 4.5 (frontier coding + agents)" + }, + { + id: "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + description: "Claude Sonnet 4.5 (complex agents + coding)" + }, + { + id: "global.anthropic.claude-haiku-4-5-20251001-v1:0", + description: "Claude Haiku 4.5 (fastest with near-frontier intelligence)" + }, + { + id: "global.anthropic.claude-opus-4-1-20250805-v1:0", + description: "Claude Opus 4.1 (agentic search + reasoning)" + }, + { + id: "global.anthropic.claude-sonnet-4-20250514-v1:0", + description: "Claude Sonnet 4.0" + }, + { + id: "anthropic.claude-3-5-haiku-20241022-v1:0", + description: "Claude 3.5 Haiku (legacy)" + }, + + // ── Meta Llama ──────────────────────────────────────────── + { + id: "us.meta.llama4-maverick-17b-instruct-v1:0", + description: "Llama 4 Maverick 17B (128 experts, 400B params, multimodal)" + }, + { + id: "us.meta.llama4-scout-17b-instruct-v1:0", + description: "Llama 4 Scout 17B (16 experts, 3.5M context)" + }, + { + id: "us.meta.llama3-3-70b-instruct-v1:0", + description: "Llama 3.3 70B Instruct" + }, + + // ── Mistral AI ──────────────────────────────────────────── + { + id: "us.mistral.mistral-large-2511-v1:0", + description: "Mistral Large 3 (flagship text, 128K context)" + }, + { + id: "us.mistral.magistral-small-2506-v1:0", + description: "Magistral Small 1.2 (reasoning, cost-effective)" + }, + + // ── DeepSeek ────────────────────────────────────────────── + { + id: "us.deepseek.r1-v1:0", + description: "DeepSeek-R1 (reasoning)" + }, + + // ── Amazon Nova ─────────────────────────────────────────── + { + id: "us.amazon.nova-pro-v1:0", + description: "Amazon Nova Pro (multimodal, balanced)" + }, + { + id: "us.amazon.nova-lite-v1:0", + description: "Amazon Nova Lite (fast, multimodal)" + }, + { + id: "us.amazon.nova-micro-v1:0", + description: "Amazon Nova Micro (text-only, cheapest)" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/chunking-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/chunking-param-types.jsonnet new file mode 100644 index 00000000..f82fa3f7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/chunking-param-types.jsonnet @@ -0,0 +1,25 @@ +// Chunk parameter type definitions + +{ + "chunk-size": { + "type": "integer", + "description": "Chunk size", + "placeholder": 2000, + "helper": "An integer, usually 2000 .. 8000", + "default": 2000, + "min": 0, + "max": 32768, + "required": true + }, + "chunk-overlap": { + "type": "integer", + "description": "Chunk overlap", + "placeholder": 50, + "helper": "An integer, usually 50 .. 100", + "default": 50, + "min": 0, + "max": 8000, + "required": true + }, +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/claude.jsonnet new file mode 100644 index 00000000..b14a317c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/claude.jsonnet @@ -0,0 +1,39 @@ +// Claude LLM Model Definitions +// Defines available models and their configurations for Anthropic's Claude + +{ + "type": "string", + "description": "LLM model to use", + "default": "claude-sonnet-4-5-20250929", + "enum": [ + { + id: "claude-opus-4-6", + description: "Claude Opus 4.6 (maximum intelligence)" + }, + { + id: "claude-opus-4-5-20251101", + description: "Claude Opus 4.5 (frontier coding + agents)" + }, + { + id: "claude-sonnet-4-5-20250929", + description: "Claude Sonnet 4.5 (complex agents + coding)" + }, + { + id: "claude-haiku-4-5-20251001", + description: "Claude Haiku 4.5 (fast)" + }, + { + id: "claude-opus-4-1-20250805", + description: "Claude Opus 4.1 (agentic search + reasoning)" + }, + { + id: "claude-sonnet-4-20250514", + description: "Claude Sonnet 4.0" + }, + { + id: "claude-3-5-haiku-20241022", + description: "Claude 3.5 Haiku" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/cohere.jsonnet new file mode 100644 index 00000000..5a5865a3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/cohere.jsonnet @@ -0,0 +1,35 @@ +// Cohere LLM Model Definitions +// Defines available models and their configurations for Cohere + +{ + "type": "string", + "description": "LLM model to use", + "default": "command-r-plus-08-2024", + "enum": [ + { + id: "command-r-plus-08-2024", + description: "Command R+ (August 2024)" + }, + { + id: "command-r-08-2024", + description: "Command R (August 2024)" + }, + { + id: "command-r-plus", + description: "Command R+ (legacy)" + }, + { + id: "command-r", + description: "Command R (legacy)" + }, + { + id: "command", + description: "Command" + }, + { + id: "command-light", + description: "Command Light" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..477e25dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/embeddings-fastembed.jsonnet @@ -0,0 +1,127 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "sentence-transformers/all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "BAAI/bge-small-en-v1.5", + "description": "bge-small-en-v1.5" + }, + { + "id": "BAAI/bge-small-zh-v1.5", + "description": "bge-small-zh-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-xs", + "description": "snowflake-arctic-embed-xs" + }, + { + "id": "jinaai/jina-embeddings-v2-small-en", + "description": "jina-embeddings-v2-small-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "snowflake/snowflake-arctic-embed-s", + "description": "snowflake-arctic-embed-s" + }, + { + "id": "BAAI/bge-small-en", + "description": "bge-small-en" + }, + { + "id": "BAAI/bge-base-en-v1.5", + "description": "bge-base-en-v1.5" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "description": "paraphrase-multilingual-MiniLM-L12-v2" + }, + { + "id": "Qdrant/clip-ViT-B-32-text", + "description": "clip-ViT-B-32-text" + }, + { + "id": "jinaai/jina-embeddings-v2-base-de", + "description": "jina-embeddings-v2-base-de" + }, + { + "id": "BAAI/bge-base-en", + "description": "bge-base-en" + }, + { + "id": "snowflake/snowflake-arctic-embed-m", + "description": "snowflake-arctic-embed-m" + }, + { + "id": "thenlper/gte-base", + "description": "gte-base" + }, + { + "id": "jinaai/jina-embeddings-v2-base-en", + "description": "jina-embeddings-v2-base-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1", + "description": "nomic-embed-text-v1" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5", + "description": "nomic-embed-text-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-m-long", + "description": "snowflake-arctic-embed-m-long" + }, + { + "id": "jinaai/jina-clip-v1", + "description": "jina-clip-v1" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + { + "id": "jinaai/jina-embeddings-v2-base-es", + "description": "jina-embeddings-v2-base-es" + }, + { + "id": "jinaai/jina-embeddings-v2-base-code", + "description": "jina-embeddings-v2-base-code" + }, + { + "id": "jinaai/jina-embeddings-v2-base-zh", + "description": "jina-embeddings-v2-base-zh" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "description": "paraphrase-multilingual-mpnet-base-v2" + }, + { + "id": "snowflake/snowflake-arctic-embed-l", + "description": "snowflake-arctic-embed-l" + }, + { + "id": "BAAI/bge-large-en-v1.5", + "description": "bge-large-en-v1.5" + }, + { + "id": "thenlper/gte-large", + "description": "gte-large" + }, + { + "id": "intfloat/multilingual-e5-large", + "description": "multilingual-e5-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/embeddings-huggingface.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/embeddings-huggingface.jsonnet new file mode 100644 index 00000000..32254a2b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/embeddings-huggingface.jsonnet @@ -0,0 +1,31 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "all-mpnet-base-v2", + "description": "all-mpnet-base-v2" + }, + { + "id": "all-distilroberta-v1", + "description": "all-distilroberta-v1" + }, + { + "id": "stsb-bert-large", + "description": "stsb-bert-large" + }, + { + "id": "sentence-camembert-large", + "description": "sentence-camembert-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/embeddings-ollama.jsonnet new file mode 100644 index 00000000..c3f29fbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/embeddings-ollama.jsonnet @@ -0,0 +1,23 @@ +// Embeddings model definitions for Ollama +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "all-minilm:latest", + "enum": [ + { + "id": "all-minilm:latest", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/googleaistudio.jsonnet new file mode 100644 index 00000000..6d454ea7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/googleaistudio.jsonnet @@ -0,0 +1,54 @@ +// Google AI Studio LLM Model Definitions +// Defines available models and their configurations for Google AI Studio + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 3 models (preview) + { + id: "gemini-3-pro-preview", + description: "Gemini 3 Pro (preview)" + }, + { + id: "gemini-3-flash-preview", + description: "Gemini 3 Flash (preview)" + }, + + // Gemini 2.5 models + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-001", + description: "Gemini 2.0 Flash" + }, + { + id: "gemini-2.0-flash-lite-001", + description: "Gemini 2.0 Flash Lite" + }, + + // Gemma models + { + id: "gemma-3-27b", + description: "Gemma 3 27B" + }, + { + id: "gemma-3n-e4b", + description: "Gemma 3n E4B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/llamafile.jsonnet new file mode 100644 index 00000000..f6480aa9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/llamafile.jsonnet @@ -0,0 +1,9 @@ +// LlamaFile LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/lmstudio.jsonnet new file mode 100644 index 00000000..1c88da3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/lmstudio.jsonnet @@ -0,0 +1,150 @@ +// LMStudio LLM Model Definitions +// Defines available models and their configurations for LMStudio + +{ + "type": "string", + "description": "LLM model to use", + "default": "llama3.1:70b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/mistral.jsonnet new file mode 100644 index 00000000..07f75b81 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/mistral.jsonnet @@ -0,0 +1,61 @@ +// Mistral LLM Model Definitions +// Defines available models and their configurations for Mistral AI + +{ + "type": "string", + "description": "LLM model to use", + "default": "mistral-medium-2508", + "enum": [ + // Featured models + { + id: "mistral-medium-2508", + description: "Mistral Medium 3.1" + }, + { + id: "mistral-large-2512", + description: "Mistral Large 3" + }, + { + id: "mistral-small-2506", + description: "Mistral Small 3.2" + }, + { + id: "ministral-14b-2512", + description: "Ministral 3 14B" + }, + { + id: "ministral-8b-2512", + description: "Ministral 3 8B" + }, + { + id: "ministral-3b-2512", + description: "Ministral 3 3B" + }, + { + id: "magistral-medium-2509", + description: "Magistral Medium 1.2 (reasoning)" + }, + { + id: "magistral-small-2509", + description: "Magistral Small 1.2 (reasoning)" + }, + { + id: "devstral-2512", + description: "Devstral 2 (code)" + }, + // Other models + { + id: "codestral-2508", + description: "Codestral (code)" + }, + { + id: "pixtral-large-2411", + description: "Pixtral Large (vision)" + }, + { + id: "open-mistral-nemo", + description: "Open Mistral Nemo" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/ollama.jsonnet new file mode 100644 index 00000000..4ef98a6a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/ollama.jsonnet @@ -0,0 +1,150 @@ +// Ollama LLM Model Definitions +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemma3:12b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/openai.jsonnet new file mode 100644 index 00000000..5039c982 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/openai.jsonnet @@ -0,0 +1,51 @@ +// OpenAI LLM Model Definitions +// Defines available models and their configurations for OpenAI's platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-5.2", + "enum": [ + { + id: "gpt-5.2-pro", + description: "GPT-5.2 Pro (maximum intelligence, slow)" + }, + { + id: "gpt-5.2", + description: "GPT-5.2 (flagship, best general-purpose)" + }, + { + id: "gpt-5.2-codex", + description: "GPT-5.2 Codex (optimized for agentic coding)" + }, + { + id: "gpt-5.1", + description: "GPT-5.1 (previous flagship)" + }, + { + id: "gpt-5", + description: "GPT-5 (previous gen reasoning)" + }, + { + id: "gpt-4.1", + description: "GPT-4.1 (coding + long context, 1M tokens)" + }, + { + id: "gpt-4.1-mini", + description: "GPT-4.1 Mini (fast + affordable)" + }, + { + id: "gpt-4.1-nano", + description: "GPT-4.1 Nano (ultra-fast, cheapest)" + }, + { + id: "gpt-4o", + description: "GPT-4o (legacy multimodal)" + }, + { + id: "gpt-4o-mini", + description: "GPT-4o Mini (legacy, budget)" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/temperature-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/temperature-param-types.jsonnet new file mode 100644 index 00000000..f5523e88 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/temperature-param-types.jsonnet @@ -0,0 +1,15 @@ +// LLM temperature definitions + +{ + "llm-temperature": { + "type": "float", + "description": "LLM temperature", + "placeholder": 0.3, + "helper": "A floating point number between 0 and 1", + "default": 0.3, + "min": 0.0, + "max": 10.0, + "required": true + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/vertexai.jsonnet new file mode 100644 index 00000000..34318967 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/vertexai.jsonnet @@ -0,0 +1,81 @@ +// VertexAI LLM Model Definitions +// Defines available models and their configurations for Google's VertexAI platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 3 models (preview) + { + id: "gemini-3-pro-preview", + description: "Gemini 3 Pro (preview)" + }, + { + id: "gemini-3-flash-preview", + description: "Gemini 3 Flash (preview)" + }, + + // Gemini 2.5 models + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-001", + description: "Gemini 2.0 Flash" + }, + { + id: "gemini-2.0-flash-lite-001", + description: "Gemini 2.0 Flash Lite" + }, + + // Gemma models + { + id: "gemma-3-27b", + description: "Gemma 3 27B" + }, + { + id: "gemma-3n-e4b", + description: "Gemma 3n E4B" + }, + + // Claude models on VertexAI + { + id: "claude-opus-4-6", + description: "Claude Opus 4.6" + }, + { + id: "claude-opus-4-5", + description: "Claude Opus 4.5" + }, + { + id: "claude-sonnet-4-5", + description: "Claude Sonnet 4.5" + }, + { + id: "claude-haiku-4-5", + description: "Claude Haiku 4.5" + }, + { + id: "claude-opus-4-1", + description: "Claude Opus 4.1" + }, + { + id: "claude-sonnet-4", + description: "Claude Sonnet 4" + }, + ], + "required": true +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/vllm.jsonnet new file mode 100644 index 00000000..120342b2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/parameters/vllm.jsonnet @@ -0,0 +1,18 @@ +// vLLM Model Definitions +// Defines available models and their configurations for vLLM +// vLLM works with any HuggingFace model. We have to use the model +// vLLM was initialised with + +{ + "type": "string", + "description": "LLM model to use", + "default": "model", + "enum": [ + // Llama 3.1 models + { + id: "model", + description: "Pre-defined model" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/profiles/memory-profile-low.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/profiles/memory-profile-low.jsonnet new file mode 100644 index 00000000..e6341062 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/profiles/memory-profile-low.jsonnet @@ -0,0 +1,169 @@ +// Low memory profile - reduces memory allocation across components. +// Include this at the END of your configuration to override default memory +// settings with lower values suitable for memory-constrained environments. +// +// Usage: {"name": "memory-profile-low", "parameters": {}} +// +// Note: This trades some performance headroom for reduced memory usage. +// Monitor for OOM errors under heavy load. + +{ + + // Override Pulsar stack memory settings + "pulsar" +: { + + // Zookeeper: 512M -> 300M + "zk-memory-limit":: "300M", + "zk-memory-reservation":: "200M", + "zk-heap":: "128m", + "zk-direct-memory":: "64m", + + // Bookie: 1024M -> 600M + "bookie-memory-limit":: "600M", + "bookie-memory-reservation":: "400M", + "bookie-heap":: "128m", + "bookie-direct-memory":: "128m", + + // Broker: 800M -> 512M + "broker-memory-limit":: "512M", + "broker-memory-reservation":: "400M", + "broker-heap":: "192m", + "broker-direct-memory":: "192m", + + // Pulsar-init: 256M -> 128M + "init-memory-limit":: "128M", + "init-memory-reservation":: "128M", + "init-heap":: "64m", + "init-direct-memory":: "64m", + + }, + + // Override Cassandra memory settings: 1000M -> 600M + "cassandra" +: { + "memory-limit":: "600M", + "memory-reservation":: "500M", + "heap":: "200M", + }, + + // Override Qdrant memory settings: 1024M -> 600M + // Also enables mmap for vectors/payloads (trades latency for memory) + "qdrant" +: { + "memory-limit":: "600M", + "memory-reservation":: "500M", + "memmap-threshold-kb":: "1", + "on-disk-payload":: "true", + }, + + // TrustGraph core services - 50% memory reservations + "api-gateway" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "chunker" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "config-svc" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "pdf-decoder" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "mcp-tool" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "mcp-server" +: { + "memory-reservation":: "128M", // 256M -> 128M + }, + + "metering" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "metering-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-store" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-manager" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "prompt" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "prompt-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-embeddings" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "librarian" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "agent-manager" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + // Graph RAG services + "kg-extract-definitions" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-relationships" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-agent" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-ontology" +: { + "memory-reservation":: "150M", // 300M -> 150M + }, + + "kg-extract-objects" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "graph-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "graph-embeddings" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + // Structured data services + "nlp-query" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "structured-query" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "structured-diag" +: { + "memory-reservation":: "48M", // 96M -> 48M + }, + + // Init service + "init-trustgraph" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/agent-kg-extract.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/agent-kg-extract.txt new file mode 100644 index 00000000..36dbd74f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/agent-kg-extract.txt @@ -0,0 +1,28 @@ +Analyze the following text and extract both entity definitions and relationships. + +For definitions, extract entities and their explanations or descriptions. +For relationships, extract subject-predicate-object triples where subjects and objects are entities, and predicates are relationship types. + +Text: {{text}} + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include a "type" field to distinguish between definitions and relationships. + +For definitions, output: +{"type": "definition", "entity": "entity_name", "definition": "definition_text"} + +For relationships, output: +{"type": "relationship", "subject": "subject_entity", "predicate": "relationship_type", "object": "object_entity_or_literal", "object-entity": true} + +Requirements: +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations, only output JSONL + +Example output: +{"type": "definition", "entity": "DNA", "definition": "Deoxyribonucleic acid, a molecule carrying genetic instructions"} +{"type": "definition", "entity": "RNA", "definition": "Ribonucleic acid, essential for coding and gene expression"} +{"type": "relationship", "subject": "DNA", "predicate": "transcribes_to", "object": "RNA", "object-entity": true} +{"type": "relationship", "subject": "DNA", "predicate": "located_in", "object": "cell nucleus", "object-entity": true} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/agent-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/agent-prompt.txt new file mode 100644 index 00000000..9155d20f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/agent-prompt.txt @@ -0,0 +1,104 @@ +# ReAct Agent System Prompt + +You are an AI assistant that uses the ReAct (Reasoning + Acting) framework to solve problems through systematic reasoning and tool use. + +## Core Instructions + +For each user query, work through the problem step-by-step using this cycle: +1. **Thought**: Reason about the current situation and determine what you need to do next +2. **Action**: Take ONE specific action using an available tool +3. Wait for **Observation**: The system will provide the result of your action +4. Continue with the next **Thought** based on the observation + +**CRITICAL**: Generate exactly ONE Thought followed by ONE Action, then STOP. Do not generate multiple Thought/Action pairs in a single response. Do not generate Observations yourself - the system will provide them. + +## Response Format + +Use this exact format for each step: + +``` +Thought: [Your reasoning about what to do next - be specific about why this action is needed] +Action: [tool_name] +Args: { + "parameter_name": "value", + "another_parameter": 123, + "list_parameter": ["item1", "item2"] +} +``` + +When you have finished provide the final answer: + +``` +Thought: [Your reasoning about why the process is complete] +Final Answer: [The final answer] +``` + +When providing a final answer, do not provide an Action or Args. + +## Action Format Rules + +1. **Tool Name**: Write "Action: " followed by the exact tool name on its own line +2. **Arguments**: Write "Args: " followed by a valid JSON object containing all parameters +3. **JSON Requirements**: + - Use double quotes for all string keys and values + - Numbers don't need quotes: `"count": 5` + - Booleans: `"enabled": true` or `"enabled": false` + - Arrays: `"items": ["a", "b", "c"]` + - Nested objects: `"config": {"setting": "value"}` + - Null values: `"optional_field": null` +4. **Required Parameters**: Include all required parameters for the tool +5. **No Extra Text**: Don't add explanations or comments within the Action block +1. **Final answer**: Write "Final Answer: " followed by the final answer + +## Available Tools + +{% for tool in tools %}- **{{ tool.name }}**: {{ tool.description }} +{% for arg in tool.arguments %} - Required: `"{{ arg.name }}"` ({{ arg.type }}): {{ arg.description }} +{% endfor %} +{% endfor %} + +## Behavior Rules + +1. **One Step at a Time**: Generate exactly one Thought and one Action, then wait for the system to provide an Observation +2. **Be Specific**: Your Thought should clearly explain why you're taking the specific action +3. **Use Context**: Build on previous Observations to inform your next steps +4. **Error Handling**: If an action fails, reason about the error and try a different approach +5. **Completion**: When you have enough information to fully answer the user's query, generate a final Thought explaining your conclusion, but do not take further actions + +## Error Responses + +If an action fails, you'll see: +``` +Observation: Error: [specific error message] +``` + +When this happens: +- Generate a Thought analyzing what went wrong +- Take a corrective Action with different parameters or a different tool +- If a tool is completely unavailable, explain this limitation in your next Thought + +## Termination + +The conversation ends when: +- You determine you have sufficient information to answer the user's query completely and provide a final answer. +- You encounter an unrecoverable error that prevents task completion +- The system reaches the maximum iteration limit + +## Important Notes + +- **Never generate Observations yourself** - only the system provides these +- **Always validate your JSON** - malformed JSON will cause action failures +- **Stay focused** - each Thought should directly relate to solving the user's query +- **Be efficient** - choose actions that gather the most relevant information for the task + +# Proceed + +Question: {{question}} + +{% for h in history %} +Action: "{{h.action}}" +Args: { +{% for k, v in h.arguments.items() %} "{{k}}": "{{v}}" +{% endfor %}} +Observation: "{{h.observation}}" +{% endfor %} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/cohere.jsonnet new file mode 100644 index 00000000..9541e4c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/cohere.jsonnet @@ -0,0 +1,42 @@ +// For Cohere. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/default-prompts.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/default-prompts.jsonnet new file mode 100644 index 00000000..3b173a5a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/default-prompts.jsonnet @@ -0,0 +1,233 @@ + +// Prompt templates. For tidy JSONNET use, don't change these templates +// here, but use over-rides in the prompt directory + +{ + + "system-template":: "You are a helpful assistant.", + + "templates":: { + + "question":: { + "prompt": "{{question}}", + }, + + "extract-definitions":: { + "prompt": importstr "extract-definitions.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "entity": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "entity", + "definition" + ] + } + }, + + "extract-relationships":: { + "prompt": importstr "extract-relationships.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "object": { + "type": "string" + }, + "object-entity": { + "type": "boolean" + } + }, + "required": [ + "subject", + "predicate", + "object", + "object-entity" + ] + } + }, + + "extract-topics":: { + "prompt": importstr "extract-topics.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "topic": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "topic", + "definition" + ] + } + }, + + "extract-rows":: { + "prompt": importstr "extract-rows.txt", + "response-type": "jsonl", + }, + + "kg-prompt":: { + "prompt": importstr "kg-prompt.txt", + "response-type": "text", + }, + + "document-prompt":: { + "prompt": importstr "document-prompt.txt", + "response-type": "text", + }, + + "agent-react":: { + "prompt": importstr "agent-prompt.txt", + "response-type": "text" + }, + + "agent-kg-extract":: { + "prompt": importstr "agent-kg-extract.txt", + "response-type": "jsonl", + "object-schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { "const": "definition" }, + "entity": { "type": "string" }, + "definition": { "type": "string" } + }, + "required": ["type", "entity", "definition"] + }, + { + "type": "object", + "properties": { + "type": { "const": "relationship" }, + "subject": { "type": "string" }, + "predicate": { "type": "string" }, + "object": { "type": "string" }, + "object-entity": { "type": "boolean" } + }, + "required": ["type", "subject", "predicate", "object"] + } + ] + } + }, + + "schema-selection":: { + "prompt": importstr "schema-selection.txt", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "An array of schema names that are relevant to answering the given question" + } + }, + + "graphql-generation":: { + "prompt": importstr "graphql-generation.txt", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The GraphQL query string generated to answer the question" + }, + "variables": { + "type": "object", + "description": "Object containing any GraphQL variables needed for the query", + "additionalProperties": true + }, + "confidence": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "description": "Float between 0.0-1.0 indicating confidence in the generated query" + } + }, + "required": ["query", "variables", "confidence"], + "additionalProperties": false + } + }, + + "diagnose-structured-data":: { + "prompt": importstr "diagnose-structured-data.txt", + "response-type": "json", + }, + + "diagnose-xml":: { + "prompt": importstr "diagnose-xml.txt", + "response-type": "json", + }, + "diagnose-json":: { + "prompt": importstr "diagnose-json.txt", + "response-type": "json", + }, + "diagnose-csv":: { + "prompt": importstr "diagnose-csv.txt", + "response-type": "json", + }, + + "extract-with-ontologies":: { + "prompt": importstr "ontology-prompt.txt", + "response-type": "jsonl", + "object-schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { "const": "entity" }, + "entity": { "type": "string" }, + "entity_type": { "type": "string" } + }, + "required": ["type", "entity", "entity_type"] + }, + { + "type": "object", + "properties": { + "type": { "const": "relationship" }, + "subject": { "type": "string" }, + "subject_type": { "type": "string" }, + "relation": { "type": "string" }, + "object": { "type": "string" }, + "object_type": { "type": "string" } + }, + "required": ["type", "subject", "subject_type", "relation", "object", "object_type"] + }, + { + "type": "object", + "properties": { + "type": { "const": "attribute" }, + "entity": { "type": "string" }, + "entity_type": { "type": "string" }, + "attribute": { "type": "string" }, + "value": { "type": "string" } + }, + "required": ["type", "entity", "entity_type", "attribute", "value"] + } + ] + } + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-csv.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-csv.txt new file mode 100644 index 00000000..ad9725ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-csv.txt @@ -0,0 +1,370 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in CSV processing and delimiter-separated value formats. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured CSV data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## CSV Processing Expertise + +When working with CSV data, you must: + +1. **Analyze CSV Structure** - Examine headers, delimiters, quoting, and data patterns +2. **Identify Column Mappings** - Map source column names to target fields +3. **Handle Complex CSV Patterns** - Support various CSV formats including: + - Standard comma-separated values: `name,age,city` + - Alternative delimiters: tab-separated (TSV), pipe-separated, semicolon-separated + - Quoted fields with embedded delimiters: `"Last, First",25,"New York, NY"` + - Headers with spaces or special characters: `"Customer Name","Order Date","Total Amount"` + - Files with or without headers + - Multi-line fields with embedded newlines + +## CSV Format Configuration Guidelines + +For CSV format configurations, use these patterns: + +**Basic CSV Configuration:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "skip_rows": 0 + } + } +} +``` + +**Advanced CSV Options:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": "\t", // Tab-separated + "quote_char": "\"", + "escape_char": "\\", + "has_header": true, + "skip_rows": 2, // Skip metadata rows + "null_values": ["", "NULL", "N/A"], + "trim_whitespace": true, + "skip_blank_lines": true + } + } +} +``` + +**CRITICAL: Source Field Names in Mappings** + +When processing CSV files, the parser uses column headers (if present) or generates column indices as field names. Your source field names in mappings must match exactly: + +**CORRECT Example with Headers:** +CSV file: +```csv +Customer Name,Order Date,Total Amount,Status +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "Customer Name": "John Smith", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Customer Name", // βœ… Correct - matches header exactly + "source_field": "Order Date", // βœ… Correct - matches header exactly + "source_field": "Total Amount", // βœ… Correct - matches header exactly + "source_field": "Status" // βœ… Correct - matches header exactly +} +``` + +**CORRECT Example without Headers:** +CSV file without headers uses column indices: +```csv +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "0": "John Smith", + "1": "2024-01-15", + "2": "1000.50", + "3": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "0", // βœ… Correct - first column + "source_field": "1", // βœ… Correct - second column + "source_field": "2", // βœ… Correct - third column + "source_field": "3" // βœ… Correct - fourth column +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - Delimiter character (comma, tab, pipe, semicolon, etc.) + - Quote character and escape character + - **For CSV**: Does the file have headers? Sample structure + - Text encoding (UTF-8, Windows-1252, etc.) + - Any rows to skip (metadata, blank lines) + - How null/empty values are represented + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source column β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Date/time format conversions + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + - Row-level validation rules + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## CSV Structure Analysis + +When presented with CSV data, analyze: + +1. **Delimiter Detection**: What character separates the fields? +2. **Header Presence**: Does the first row contain column names? +3. **Quote Pattern**: Are fields quoted? What quote character is used? +4. **Data Types**: What types are present in each column? +5. **Null Representation**: How are empty/null values represented? +6. **Special Characters**: Are there embedded commas, quotes, or newlines? +7. **Encoding Issues**: Are there any character encoding problems? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + // CSV-specific parsing options + // delimiter, quote_char, has_header, skip_rows, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `split`, `strip_quotes` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_number`, `parse_currency` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `clean_whitespace`, `normalize_encoding` + +**Date/Time Operations:** +- `parse_date`, `format_date`, `date_component` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `numeric_range`, `date_range` + +## CSV-Specific Best Practices + +1. **Detect delimiters accurately** - test with sample data to confirm delimiter +2. **Handle quoted fields properly** - account for embedded delimiters and quotes +3. **Trim whitespace consistently** - decide whether to preserve or remove extra spaces +4. **Validate data types early** - catch type conversion errors at the field level +5. **Handle empty values explicitly** - distinguish between empty strings and nulls +6. **Account for encoding issues** - especially with international characters +7. **Validate row structure** - ensure consistent column counts across rows + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle CSV-specific edge cases** like malformed quotes, inconsistent delimiters +10. **Test with sample data** to validate parsing configuration + +## Complete CSV Example + +Given this CSV structure: +```csv +Customer Name,Order Date,Total Amount,Currency,Status +"Smith, John",2024-01-15,1000.50,USD,Active +"Doe, Jane",2024-01-16,750.25,EUR,Pending +"Johnson, Bob",2024-01-17,,USD,Cancelled +``` + +The parser will: +1. Use `delimiter: ","` and `quote_char: "\""` to parse fields correctly +2. Use `has_header: true` to treat first row as column names +3. Create this parsed data structure for each record: + ```json + { + "Customer Name": "Smith, John", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Currency": "USD", + "Status": "Active" + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "trim_whitespace": true, + "null_values": ["", "NULL"] + } + }, + "mappings": [ + { + "source_field": "Customer Name", // βœ… Matches header exactly + "target_field": "customer_name", + "transforms": [{"type": "trim"}] + }, + { + "source_field": "Order Date", // βœ… Matches header exactly + "target_field": "order_date", + "transforms": [{"type": "to_date", "format": "YYYY-MM-DD"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "Total Amount", // βœ… Matches header exactly + "target_field": "amount", + "transforms": [ + {"type": "to_float"}, + {"type": "default", "value": 0.0} + ] + }, + { + "source_field": "Currency", // βœ… Matches header exactly + "target_field": "currency_code", + "transforms": [{"type": "upper"}], + "validation": [{"type": "in_list", "values": ["USD", "EUR", "GBP"]}] + }, + { + "source_field": "Status", // βœ… Matches header exactly + "target_field": "order_status", + "transforms": [{"type": "lower"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the column headers exactly, or use column indices (0, 1, 2, etc.) for files without headers.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the CSV structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to delimiter detection, header identification, quoting patterns, and data type inference: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-json.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-json.txt new file mode 100644 index 00000000..2fe6341d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-json.txt @@ -0,0 +1,327 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in JSON processing and JSONPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured JSON data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## JSON Processing Expertise + +When working with JSON data, you must: + +1. **Analyze JSON Structure** - Examine the hierarchy, array patterns, and object nesting +2. **Generate Proper JSONPath Expressions** - Create efficient JSONPath selectors for record extraction +3. **Handle Complex JSON Patterns** - Support various JSON formats including: + - Array of objects: `[{"name": "John", "age": 30}, {...}]` + - Nested object arrays: `{"data": {"records": [{"id": 1}, {...}]}}` + - Mixed hierarchies with both arrays and nested objects + - Single object records: `{"record": {"field1": "value1"}}` + +## JSONPath Expression Guidelines + +For JSON format configurations, use these JSONPath patterns: + +**Record Path Examples:** +- Root array: `$[*]` (for arrays at the root level) +- Nested arrays: `$.data.records[*]` or `$.response.items[*]` +- Single object: `$.record` (when there's one record per file) +- Deep nesting: `$.data.results.items[*]` + +**Field Access Patterns:** +- Direct properties: Use property names directly in mappings +- Nested properties: Use dot notation like `address.street` or `contact.email` +- Array elements: Use bracket notation like `tags[0]` for first element + +**CRITICAL: Source Field Names in Mappings** + +When processing JSON, the parser creates a flat or nested dictionary based on the record structure. Your source field names in mappings must match the actual property names in the parsed records: + +**CORRECT Example:** +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "metadata": { + "source": "UN", + "year": 2024 + } +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches property name + "source_field": "Trade (USD)", // βœ… Correct - matches property name + "source_field": "metadata.source", // βœ… Correct - nested property access + "source_field": "metadata.year" // βœ… Correct - nested property access +} +``` + +**JSON Format Configuration Template:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$[*]", // JSONPath to extract records + "flatten_nested": true, // Whether to flatten nested objects + "array_handling": "expand" // How to handle arrays: expand, first, concat + } + } +} +``` + +**Alternative JSON Options:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.items[*]", // For nested array structures + "flatten_nested": false, // Keep nested structure + "null_value_handling": "skip" // skip, empty_string, or preserve + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - JSON structure type (array of objects, nested objects, single records) + - **For JSON**: Sample structure, nesting patterns, array locations + - Sample data or field descriptions + - Any format-specific details (encoding, special null handling, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Nested object flattening requirements + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or null values + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## JSON Structure Analysis + +When presented with JSON data, analyze: + +1. **Root Structure**: Is it an array, object, or nested structure? +2. **Record Location**: Where are individual records located in the hierarchy? +3. **Field Pattern**: How are field names and values structured? + - Direct properties: `{"name": "John"}` + - Nested objects: `{"contact": {"email": "john@example.com"}}` + - Arrays: `{"tags": ["red", "blue"]}` + - Mixed types: `{"data": [{"id": 1, "details": {"name": "John"}}]}` +4. **Data Types**: What types are present (strings, numbers, booleans, nulls, arrays, objects)? +5. **Hierarchy Depth**: How deeply nested are the records and fields? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + // JSON-specific parsing options + // record_path (JSONPath), flatten_nested, array_handling, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `flatten_object`, `extract_array_element`, `join_array` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## JSON-Specific Best Practices + +1. **Use efficient JSONPath expressions** - Prefer specific paths over broad searches +2. **Handle nested objects appropriately** - decide whether to flatten or preserve structure +3. **Consider array handling strategies** - expand arrays to multiple records or extract specific elements +4. **Account for null vs undefined** values in field mappings +5. **Handle mixed data types** within the same field across records +6. **Use appropriate flattening** for deeply nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle JSON-specific edge cases** like empty arrays, null objects, mixed types + +## Complete JSON Example + +Given this JSON structure: +```json +{ + "data": { + "records": [ + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata": { + "source": "World Bank", + "confidence": 0.95 + } + } + ] + } +} +``` + +The parser will: +1. Use `record_path: "$.data.records[*]"` to extract record objects from the array +2. Create this parsed data structure for each record: + ```json + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata.source": "World Bank", // If flattened + "metadata.confidence": 0.95 // If flattened + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.records[*]", + "flatten_nested": true, + "array_handling": "expand" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches property name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches property name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches property name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + }, + { + "source_field": "metadata.source", // βœ… Flattened nested field + "target_field": "data_source" + } + ] +} +``` + +**KEY RULE: source_field names must match the actual property names in the parsed JSON records, using dot notation for nested properties when flattened.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the JSON structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to JSON hierarchy, object patterns, array structures, and generate appropriate JSONPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-structured-data.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-structured-data.txt new file mode 100644 index 00000000..84c4b8be --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-structured-data.txt @@ -0,0 +1,309 @@ + +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` (will be converted to relative paths automatically) +- With namespaces: `//ns:record` or `//soap:Body/data/record` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name + +**CRITICAL: Source Field Names in Mappings** + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary. Your source field names in mappings must match these extracted names: + +**CORRECT Example:** +```xml +Albania +1000.50 +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50" +} +``` + +So your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)" // βœ… Correct - matches parsed field name +} +``` + +**INCORRECT Example:** +```json +{ + "source_field": "Field[@name='Country or Area']", // ❌ Wrong - XPath not needed here + "source_field": "field[@name='Trade (USD)']" // ❌ Wrong - XPath not needed here +} +``` + +**XML Format Configuration Template:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name" // For value pattern + } + } +} +``` + +**Alternative XML Options:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // Direct element-based records + // No field_attribute needed for standard XML + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - File type (CSV, JSON, XML, Excel, fixed-width, etc.) + - **For XML**: Sample structure, namespace prefixes, record element patterns + - Sample data or field descriptions + - Any format-specific details (delimiters, encoding, namespaces, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Record Container**: Where are individual records located? +3. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns +4. **Namespaces**: Are there any namespace prefixes? +5. **Hierarchy Depth**: How deeply nested are the records? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "[csv|json|xml|fixed-width|excel]", + "encoding": "utf-8", + "options": { + // Format-specific parsing options + // For XML: record_path (XPath), field_attribute (if applicable) + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches +2. **Handle namespace prefixes** when present +3. **Identify field attribute patterns** correctly +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes + +## Complete XML Example + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "/ROOT/data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the extracted field names, NOT the XML element structure.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, and generate appropriate XPath expressions: + +{{sample}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-xml.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-xml.txt new file mode 100644 index 00000000..a3d8efa4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/diagnose-xml.txt @@ -0,0 +1,453 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for XML data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured XML data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + - CDATA sections and text nodes + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` +- With namespaces: `//ns:record` or `//soap:Body/data/record` +- Attribute-filtered: `//record[@type='customer']` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name like `field_attribute: "id"` or `field_attribute: "type"` + +## CRITICAL: Source Field Names in Mappings + +The behavior depends on whether you use `field_attribute`: + +### With field_attribute (Attribute-Based Fields) + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary: + +**Example:** +```xml + + Albania + 1000.50 + Active + +``` + +Parser configuration: +```json +{ + "record_path": "//record", + "field_attribute": "name" +} +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)", // βœ… Correct - matches parsed field name + "source_field": "Status" // βœ… Correct - matches parsed field name +} +``` + +### Without field_attribute (Element-Based Fields) + +When NOT using `field_attribute`, use direct element names or XPath expressions: + +**Example:** +```xml + + Albania + 1000.50 + Active + + UN + 2024 + + +``` + +Parser configuration: +```json +{ + "record_path": "//record" + // No field_attribute specified +} +``` + +Your mappings should use: +```json +{ + "source_field": "country", // βœ… Direct element name + "source_field": "trade_amount", // βœ… Direct element name + "source_field": "metadata/source", // βœ… Nested element path + "source_field": "metadata/year" // βœ… Nested element path +} +``` + +## XML Format Configuration Templates + +**For Attribute-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name", // Extract field names from 'name' attribute + "namespace_prefixes": { // Optional: define namespaces + "ns": "http://example.com/namespace" + } + } + } +} +``` + +**For Element-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // XPath to find record elements + "preserve_namespaces": true, // Optional: keep namespace info + "ignore_attributes": false // Optional: include element attributes + } + } +} +``` + +**For Complex XML with Namespaces:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//soap:Body//data:record", + "namespace_prefixes": { + "soap": "http://schemas.xmlsoap.org/soap/envelope/", + "data": "http://example.com/data" + }, + "field_attribute": "name" + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **XML Structure Details** + - Sample XML structure or schema + - Root element and record location + - Field organization (elements vs attributes) + - Namespace declarations and prefixes + - Text encoding (UTF-8, ISO-8859-1, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Namespaces**: Are there namespace declarations? What prefixes are used? +3. **Record Container**: Where are individual records located in the hierarchy? +4. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns: `John` +5. **Data Location**: Are values in element text, attributes, or both? +6. **Hierarchy Depth**: How deeply nested are the records? +7. **Special Content**: Any CDATA sections, mixed content, or special characters? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + // XML-specific parsing options + // record_path (XPath), field_attribute (if applicable), namespace_prefixes + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `strip_cdata`, `decode_html_entities` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_xml_date`, `parse_iso_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `extract_attribute`, `get_text_content` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `valid_xml_name`, `namespace_check` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches like `//` when possible +2. **Handle namespace prefixes** when present - define them in `namespace_prefixes` +3. **Identify field attribute patterns** correctly - determine if using `field_attribute` +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures +7. **Handle CDATA sections** and special characters properly +8. **Preserve or normalize whitespace** as appropriate +9. **Consider XML schema validation** if XSD is available + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle XML-specific edge cases** like empty elements, mixed content, processing instructions + +## Complete XML Examples + +### Example 1: Attribute-Based Fields + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "//data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +### Example 2: Element-Based Fields + +Given this XML structure: +```xml + + + John Smith + john@example.com +
+ 123 Main St + New York +
+ + + 456 + 100.50 + + +
+
+``` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer" + } + }, + "mappings": [ + { + "source_field": "name", // βœ… Direct element name + "target_field": "customer_name" + }, + { + "source_field": "email", // βœ… Direct element name + "target_field": "email_address" + }, + { + "source_field": "address/street", // βœ… Nested element path + "target_field": "street_address" + }, + { + "source_field": "address/city", // βœ… Nested element path + "target_field": "city" + }, + { + "source_field": "@id", // βœ… Attribute reference + "target_field": "customer_id", + "transforms": [{"type": "to_int"}] + } + ] +} +``` + +**KEY RULES:** +- With `field_attribute`: source_field names must match the extracted attribute values +- Without `field_attribute`: use direct element names, nested paths, or XPath expressions +- Use `@attribute` notation for XML element attributes + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, namespace usage, and generate appropriate XPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/document-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/document-prompt.txt new file mode 100644 index 00000000..5e5d65ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/document-prompt.txt @@ -0,0 +1,7 @@ +Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + +Here is the context: +{{documents}} + +Use only the provided knowledge statements to respond to the following: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-definitions.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-definitions.txt new file mode 100644 index 00000000..410ecddf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-definitions.txt @@ -0,0 +1,28 @@ + +Study the following text and derive definitions for any discovered entities. +Do not provide definitions for entities whose definitions are incomplete +or unknown. + +Output each definition as a separate JSON object on its own line (JSONL format). +Each object must have fields: +- entity: the name of the entity +- definition: English text which defines the entity + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not use special characters in the definition text +- Do not include null or unknown definitions + +Example output format: +{"entity": "photosynthesis", "definition": "The process by which plants convert sunlight into energy"} +{"entity": "chlorophyll", "definition": "Green pigment in plants that absorbs light"} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-relationships.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-relationships.txt new file mode 100644 index 00000000..31bcd2d4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-relationships.txt @@ -0,0 +1,28 @@ + +Study the following text and derive entity relationships. For each +relationship, derive the subject, predicate and object of the relationship. + +Output each relationship as a separate JSON object on its own line (JSONL format). +Each object must have fields: +- subject: the subject of the relationship +- predicate: the predicate +- object: the object of the relationship +- object-entity: false if the object is a simple data type (name, value, date). true if it is an entity. + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not use special characters in the text fields + +Example output format: +{"subject": "Earth", "predicate": "orbits", "object": "Sun", "object-entity": true} +{"subject": "Earth", "predicate": "has diameter", "object": "12742 km", "object-entity": false} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-rows.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-rows.txt new file mode 100644 index 00000000..7cea8b7d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-rows.txt @@ -0,0 +1,27 @@ + +Study the following text and derive objects which match the schema provided. + +Output each discovered object as a separate JSON object on its own line (JSONL format). +Each object's fields must carry the name field specified in the schema. + + + +{{schema}} + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object matching the schema +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations + +Example output format (assuming schema has fields: name, age, city): +{"name": "Alice", "age": 30, "city": "London"} +{"name": "Bob", "age": 25, "city": "Paris"} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-topics.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-topics.txt new file mode 100644 index 00000000..bb4b1193 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/extract-topics.txt @@ -0,0 +1,26 @@ +You are a helpful assistant that performs information extraction tasks for a provided text. +Read the provided text. You will identify topics and their definitions. + +Reading Instructions: +- Ignore document formatting in the provided text. +- Study the provided text carefully. + +Here is the text: +{{text}} + +Response Instructions: +- Output each topic as a separate JSON object on its own line (JSONL format) +- Each object must have keys "topic" and "definition" +- Do not respond with special characters +- Return only topics that are concepts and unique to the provided text + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not write any additional text or explanations + +Example output format: +{"topic": "machine learning", "definition": "A subset of AI that enables systems to learn from data"} +{"topic": "neural network", "definition": "A computing system inspired by biological neural networks"} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/gemini.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/gemini.jsonnet new file mode 100644 index 00000000..b9a1e0c0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/gemini.jsonnet @@ -0,0 +1,42 @@ +// For VertexAI Gemini. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/graphql-generation.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/graphql-generation.txt new file mode 100644 index 00000000..58af61dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/graphql-generation.txt @@ -0,0 +1,59 @@ +You are a GraphQL query generation expert. Given a natural language question and provided database schemas, generate a valid GraphQL query embedded in valid JSON. + +## Question: +{{ question }} + +## Provided Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %}- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif %}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif %}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ field.enum_values|join(', ') }}]{% endif %} +{% endfor %} +{% endfor %} + +## GraphQL Query Rules: +1. Use the schema names as GraphQL query fields (e.g., `customers`, `orders`) +2. Apply filters using the `where` parameter with nested filter objects +3. Available filter operators per field type: + - String fields: `eq`, `contains`, `startsWith`, `endsWith`, `in`, `not`, `not_in` + - Integer/Float fields: `eq`, `gt`, `gte`, `lt`, `lte`, `in`, `not`, `not_in` +4. Use `order_by` for sorting (field name as string) +5. Use `direction` for sort direction: `ASC` or `DESC` +6. Use `limit` to restrict number of results +7. Select specific fields in the query body + +## Task Instructions: +1. Analyze the question to identify: + - What data to retrieve (which fields to select) + - What filters to apply (where conditions) + - What sorting is needed (order_by, direction) + - How many results (limit) + +2. Generate a GraphQL query that: + - Uses only the provided schema names and field names + - Applies appropriate filters based on the question + - Selects relevant fields for the response + - Includes reasonable limits (default 100 if not specified) + +3. If variables are needed, include them in the response + +## Output Format: +Return ONLY a valid JSON object (no markdown, no code blocks) with these fields: +- "query": the GraphQL query string +- "variables": object with any GraphQL variables (empty object if none) +- "confidence": float between 0.0-1.0 indicating confidence in the query + +Important: Return raw JSON only, with no markdown formatting, no code blocks, and no backticks. + +## Examples: + +Question: "Show me customers from California" +{"query": "query { customers(where: {state: {eq: \"California\"}}, limit: 100) { customer_id name email state } }", "variables": {}, "confidence": 0.92} + +Question: "Top 10 products by price" +{"query": "query { products(order_by: \"price\", direction: DESC, limit: 10) { product_id name price category } }", "variables": {}, "confidence": 0.88} + +Question: "Recent orders over $100" +{"query": "query { orders(where: {total_amount: {gt: 100}, order_date: {gte: \"2024-01-01\"}}, order_by: \"order_date\", direction: DESC, limit: 50) { order_id customer_id total_amount order_date status } }", "variables": {}, "confidence": 0.96} + +Now generate the GraphQL query for the question above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/kg-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/kg-prompt.txt new file mode 100644 index 00000000..431c0d73 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/kg-prompt.txt @@ -0,0 +1,8 @@ +Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + +Here's the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}}) +{%endfor%} + +Use only the provided knowledge statements to respond to the following: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/mixtral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/mixtral.jsonnet new file mode 100644 index 00000000..cd56e7ef --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/mixtral.jsonnet @@ -0,0 +1,42 @@ +// For Mixtral. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/ontology-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/ontology-prompt.txt new file mode 100644 index 00000000..1b451d9c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/ontology-prompt.txt @@ -0,0 +1,82 @@ +You are a knowledge extraction expert. Your task is to find entities, relationships, and attributes in text based on a provided schema. + +## Entity Types + +These are the types of entities you should look for: + +{% for class_id, class_def in classes.items() %} +- **{{class_id}}**{% if class_def.subclass_of %} (subclass of {{class_def.subclass_of}}){% endif %}{% if class_def.comment %}: {{class_def.comment}}{% endif %} +{% endfor %} + +## Relationships + +These relationships connect entities to other entities: + +{% for prop_id, prop_def in object_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Attributes + +These attributes describe entity properties (text, numbers, etc.): + +{% for prop_id, prop_def in datatype_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Text to Analyze + +{{text}} + +## Your Task + +Extract the following from the text above: + +1. **Entities**: Things mentioned in the text and their types +2. **Relationships**: How entities relate to each other +3. **Attributes**: Properties of entities (like quantities, descriptions, etc.) + +## Output Format + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include a "type" field to distinguish between entities, relationships, and attributes. + +For entities, output: +{"type": "entity", "entity": "entity name as it appears in text", "entity_type": "EntityType"} + +For relationships, output: +{"type": "relationship", "subject": "subject entity name", "subject_type": "SubjectType", "relation": "relationship_name", "object": "object entity name", "object_type": "ObjectType"} + +For attributes, output: +{"type": "attribute", "entity": "entity name", "entity_type": "EntityType", "attribute": "attribute_name", "value": "literal value"} + +## Important Rules + +1. **Entity names**: Use the exact text as it appears (e.g., "Cornish pasty", "beef") +2. **Types**: Use the EXACT type identifiers from the schema above (e.g., "fo/Recipe", "fo/Food") +3. **Relationships**: Use the EXACT relationship names from the schema (e.g., "fo/has_ingredient") +4. **Attributes**: Use the EXACT attribute names from the schema (e.g., "fo/serves") +5. **No array brackets**: Output one JSON object per line, not an array +6. **No markdown**: Do not wrap output in code blocks + +## Requirements + +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting, code blocks, or prefixes +- Do not provide explanations, only output JSONL + +## Example + +Input text: "Cornish pasty is a savory pastry filled with beef and potatoes. This recipe serves 4 people." + +Expected output: +{"type": "entity", "entity": "Cornish pasty", "entity_type": "fo/Recipe"} +{"type": "entity", "entity": "beef", "entity_type": "fo/Food"} +{"type": "entity", "entity": "potatoes", "entity_type": "fo/Food"} +{"type": "relationship", "subject": "Cornish pasty", "subject_type": "fo/Recipe", "relation": "fo/has_ingredient", "object": "beef", "object_type": "fo/Food"} +{"type": "relationship", "subject": "Cornish pasty", "subject_type": "fo/Recipe", "relation": "fo/has_ingredient", "object": "potatoes", "object_type": "fo/Food"} +{"type": "attribute", "entity": "Cornish pasty", "entity_type": "fo/Recipe", "attribute": "fo/serves", "value": "4 people"} + +Now extract entities, relationships, and attributes from the text above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/openai.jsonnet new file mode 100644 index 00000000..5d232337 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/openai.jsonnet @@ -0,0 +1,42 @@ +// For OpenAI LLMs. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/schema-selection.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/schema-selection.txt new file mode 100644 index 00000000..39b180e5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/schema-selection.txt @@ -0,0 +1,25 @@ +You are a database schema selection expert. Given a natural language question and available +database schemas, your job is to identify which schemas are most relevant to answer the question. + +## Available Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}): {{ field.description }} +{% endfor %} + +{% endfor %} + +## Question: +{{ question }} + +## Instructions: +1. Analyze the question to understand what data is being requested +2. Examine each schema to understand what data it contains +3. Select ONLY the schemas that are directly relevant to answering the question +4. Return your answer as a JSON array of schema names + +## Response Format: +Return ONLY a JSON array of schema names, nothing else. +Example: ["customers", "orders", "products"] diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/slm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/slm.jsonnet new file mode 100644 index 00000000..48eb96d0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/prompts/slm.jsonnet @@ -0,0 +1,44 @@ +// For SLM. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/pulsar/pulsar-manager.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/pulsar/pulsar-manager.jsonnet new file mode 100644 index 00000000..c1ca4515 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/pulsar/pulsar-manager.jsonnet @@ -0,0 +1,40 @@ +local images = import "values/images.jsonnet"; + +{ + + "pulsar" +: { + + create:: function(engine) + +// FIXME: Should persist something? +// local volume = engine.volume(...) + + local container = + engine.container("pulsar") + .with_image(images.pulsar_manager) + .with_environment({ + SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties", + }) + .with_limits("0.5", "1.4G") + .with_reservations("0.1", "1.4G") + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2"); + + local containerSet = engine.containers( + "pulsar", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2); + + engine.resources([ + containerSet, + service, + ]) + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/pulsar/pulsar.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/pulsar/pulsar.jsonnet new file mode 100644 index 00000000..d417322d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/pulsar/pulsar.jsonnet @@ -0,0 +1,222 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +// This is a Pulsar configuration. Non-standalone mode so we deploy +// individual components: bookkeeper, broker and zookeeper. +// +// This also deploys the TrustGraph 'admin' container which initialises +// TrustGraph-specific namespaces etc. + +{ + + "pulsar" +: { + + // Zookeeper memory settings (can be overridden by memory-profile) + "zk-memory-limit":: "512M", + "zk-memory-reservation":: "512M", + "zk-heap":: "256m", + "zk-direct-memory":: "256m", + + // Bookie memory settings (can be overridden by memory-profile) + "bookie-memory-limit":: "1024M", + "bookie-memory-reservation":: "1024M", + "bookie-heap":: "256m", + "bookie-direct-memory":: "256m", + + // Broker memory settings (can be overridden by memory-profile) + "broker-memory-limit":: "800M", + "broker-memory-reservation":: "800M", + "broker-heap":: "384m", + "broker-direct-memory":: "384m", + + // Pulsar-init memory settings (can be overridden by memory-profile) + "init-memory-limit":: "256M", + "init-memory-reservation":: "256M", + "init-heap":: "128m", + "init-direct-memory":: "128m", + + create:: function(engine) + + // Capture memory settings into locals (self refers to pulsar object here) + local zkMemLimit = self["zk-memory-limit"]; + local zkMemReserv = self["zk-memory-reservation"]; + local zkHeap = self["zk-heap"]; + local zkDirect = self["zk-direct-memory"]; + + local bookieMemLimit = self["bookie-memory-limit"]; + local bookieMemReserv = self["bookie-memory-reservation"]; + local bookieHeap = self["bookie-heap"]; + local bookieDirect = self["bookie-direct-memory"]; + + local brokerMemLimit = self["broker-memory-limit"]; + local brokerMemReserv = self["broker-memory-reservation"]; + local brokerHeap = self["broker-heap"]; + local brokerDirect = self["broker-direct-memory"]; + + local initMemLimit = self["init-memory-limit"]; + local initMemReserv = self["init-memory-reservation"]; + local initHeap = self["init-heap"]; + local initDirect = self["init-direct-memory"]; + + // Zookeeper volume + local zkVolume = engine.volume("zookeeper").with_size("1G"); + + // Zookeeper container + local zkContainer = + engine.container("zookeeper") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper" + ]) + .with_limits("1", zkMemLimit) + .with_reservations("0.05", zkMemReserv) + .with_user("0:1000") + .with_volume_mount(zkVolume, "/pulsar/data/zookeeper") + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + zkHeap, zkHeap, zkDirect, + ], + }) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Pulsar cluster init container + local initContainer = + engine.container("pulsar-init") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650", + ]) + .with_limits("1", initMemLimit) + .with_reservations("0.05", initMemReserv) + .with_environment({ + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + initHeap, initHeap, initDirect, + ], + }); + + + // Bookkeeper volume + local bookieVolume = engine.volume("bookie").with_size("20G"); + + // Bookkeeper container + local bookieContainer = + engine.container("bookie") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + // false ^ causes this to be a 'failure' exit. + ]) + .with_limits("1", bookieMemLimit) + .with_reservations("0.1", bookieMemReserv) + .with_user("0:1000") + .with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper") + .with_environment({ + "clusterName": "cluster-a", + "zkServers": "zookeeper:2181", + "bookieId": "bookie", + "metadataStoreUri": "metadata-store:zk:zookeeper:2181", + "advertisedAddress": "bookie", + "BOOKIE_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + bookieHeap, bookieHeap, bookieDirect, + ], + }) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker, stateless (uses ZK and Bookkeeper for state) + local brokerContainer = + engine.container("pulsar") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" + ]) + .with_limits("1", brokerMemLimit) + .with_reservations("0.1", brokerMemReserv) + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "zookeeperServers": "zookeeper:2181", + "clusterName": "cluster-a", + "managedLedgerDefaultEnsembleSize": "1", + "managedLedgerDefaultWriteQuorum": "1", + "managedLedgerDefaultAckQuorum": "1", + "advertisedAddress": "pulsar", + "advertisedListeners": "external:pulsar://pulsar:6650,localhost:pulsar://localhost:6650", + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + brokerHeap, brokerHeap, brokerDirect, + ], + }) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + // Container sets + local zkContainerSet = engine.containers( + "zookeeper", + [ + zkContainer, + ] + ); + + local initContainerSet = engine.containers( + "init-pulsar", + [ + initContainer, + ] + ); + + local bookieContainerSet = engine.containers( + "bookie", + [ + bookieContainer, + ] + ); + + local brokerContainerSet = engine.containers( + "pulsar", + [ + brokerContainer, + ] + ); + + // Zookeeper service + local zkService = + engine.service(zkContainerSet) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Bookkeeper service + local bookieService = + engine.service(bookieContainerSet) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker service + local brokerService = + engine.service(brokerContainerSet) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + engine.resources([ + zkVolume, + bookieVolume, + zkContainerSet, + initContainerSet, + bookieContainerSet, + brokerContainerSet, + zkService, + bookieService, + brokerService, + ]) + + } + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-additionals.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-additionals.jsonnet new file mode 100644 index 00000000..3e9f3481 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-additionals.jsonnet @@ -0,0 +1,134 @@ +local decode = import "decode-config.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Custom engine that collects configVolume parts +local engine = { + + // Collection of all configVolume parts + configVolumes:: [], + + // Implement all required engine methods as no-ops + container:: function(name) { + with_image:: function(x) self, + with_command:: function(x) self, + with_environment:: function(x) self, + with_limits:: function(c, m) self, + with_reservations:: function(c, m) self, + with_port:: function(src, dest, name) self, + with_volume_mount:: function(vol, mnt) self, + with_user:: function(x) self, + with_runtime:: function(x) self, + with_privileged:: function(x) self, + with_ipc:: function(x) self, + with_capability:: function(x) self, + with_device:: function(hdev, cdev) self, + with_env_var_secrets:: function(vars) self, + }, + + volume:: function(name) { + with_size:: function(size) self, + }, + + // The key method - collects configVolume parts + configVolume:: function(name, dir, parts) + local collector = self + { + configVolumes: super.configVolumes + [ + { + dir: dir, + parts: parts, + } + ] + }; + { + // Return a dummy volume that has the collector in it + name: name, + with_size:: function(size) collector, + // Provide a way to get back to the collector + getCollector:: function() collector, + }, + + secretVolume:: function(name, dir, parts) { + with_size:: function(size) self, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self, + }, + + containers:: function(name, containers) self, + + service:: function(containers) { + with_port:: function(src, dest, name) self, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self, + }, + + resources:: function(res) + // Fold over resources and collect any configVolume state + local collected = std.foldl( + function(state, r) + if std.objectHasAll(r, 'getCollector') then + // Merge the configVolumes from the volume's collector into our state + local volumeCollector = r.getCollector(); + state + { + configVolumes: state.configVolumes + volumeCollector.configVolumes + } + else + state, + res, + self + ); + collected, +}; + +// Execute all component create() functions with our collecting engine +// Note: create:: is a hidden field, so we must use objectHasAll not objectHas +local result = std.foldl( + function(state, p) + if std.objectHasAll(p, 'create') then + // Pattern has create directly - call it + p.create(state) + else + state, + std.objectValues(patterns), + engine +); + +// Debug: show what we collected +local debug = { + numPatterns: std.length(std.objectValues(patterns)), + numConfigVolumes: std.length(result.configVolumes), +}; + +// Transform collected data into output format +local allFiles = std.flattenArrays([ + [ + { + // Remove trailing slash from dir to avoid double slashes + path: std.join("/", [std.rstripChars(cv.dir, "/"), filename]), + content: cv.parts[filename] + } + for filename in std.objectFields(cv.parts) + ] + for cv in result.configVolumes +]); + +// Deduplicate by path - use a map to keep only unique paths +local uniqueMap = std.foldl( + function(acc, item) acc + { [item.path]: item }, + allFiles, + {} +); + +// Convert back to array +local additionals = std.objectValues(uniqueMap); + +// Output the array +additionals diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-aks-k8s.jsonnet new file mode 100644 index 00000000..bcec1cbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-aks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/aks-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-docker-compose.jsonnet new file mode 100644 index 00000000..7f0dbabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-docker-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/docker-compose.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-eks-k8s.jsonnet new file mode 100644 index 00000000..f9de59cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-eks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/eks-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-gcp-k8s.jsonnet new file mode 100644 index 00000000..50037a5c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-gcp-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/gcp-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-minikube-k8s.jsonnet new file mode 100644 index 00000000..6fa64706 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-minikube-k8s.jsonnet @@ -0,0 +1,26 @@ + +local engine = import "../engine/minikube-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +// Extract resources using the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-noop.jsonnet new file mode 100644 index 00000000..3ad735d9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-noop.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/noop.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-ovh-k8s.jsonnet new file mode 100644 index 00000000..92f61041 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-ovh-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/ovh-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-podman-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-podman-compose.jsonnet new file mode 100644 index 00000000..7f0dbabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-podman-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/docker-compose.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-scw-k8s.jsonnet new file mode 100644 index 00000000..4f6c1d7a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-scw-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/scw-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-tg-configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-tg-configuration.jsonnet new file mode 100644 index 00000000..59b4732a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/config-to-tg-configuration.jsonnet @@ -0,0 +1,15 @@ + +local engine = import "../engine/noop.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract configuration directly from patterns +patterns.configuration.configuration + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/decode-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/decode-config.jsonnet new file mode 100644 index 00000000..759513c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/renderers/decode-config.jsonnet @@ -0,0 +1,31 @@ + +local components = import "../components.jsonnet"; + +local apply = function(p, components) + + local base = { + + with:: function(k, v) self + { + [k] +:: v + }, + + with_params:: function(pars) + self + std.foldl( + function(obj, par) obj.with(par.key, par.value), + std.objectKeysValues(pars), + self + ), + + }; + + local component = base + components[p.name]; + + component.with_params(p.parameters); + +local decode = function(config) + local add = function(state, c) state + apply(c, components); + local patterns = std.foldl(add, config, {}); + patterns; + +decode + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/row-store/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/row-store/cassandra.jsonnet new file mode 100644 index 00000000..f90526bc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/row-store/cassandra.jsonnet @@ -0,0 +1,77 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "backends/cassandra.jsonnet"; + +cassandra + { + + "store-rows" +: { + + create:: function(engine) + + local container = + engine.container("store-rows") + .with_image(images.trustgraph_flow) + .with_command([ + "rows-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-rows", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-rows" +: { + + create:: function(engine) + + local container = + engine.container("query-rows") + .with_image(images.trustgraph_flow) + .with_command([ + "rows-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-rows", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/config-composer.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/config-composer.jsonnet new file mode 100644 index 00000000..e93ff044 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/config-composer.jsonnet @@ -0,0 +1,97 @@ +// Configuration Composer Module +// Orchestrates the complete configuration building process +// Combines all components into the final TrustGraph configuration + +local flow_builder = import "flow-builder.jsonnet"; +local interface_builder = import "interface-builder.jsonnet"; + +{ + // Main function to build the complete configuration + build: function(config_spec) + // Extract configuration parameters + local flow_blueprints = config_spec.flow_blueprints; + local default_flow_blueprint = config_spec.default_flow_blueprint; + local default_flow_id = config_spec.default_flow_id; + local flow_init_parameters = config_spec.flow_init_parameters; + + // Build all processors for the default flow + local blueprint_processors = flow_builder.build_blueprint_processors( + flow_blueprints, + default_flow_blueprint, + flow_init_parameters + ); + + local flow_processors = flow_builder.build_flow_processors( + flow_blueprints, + default_flow_blueprint, + default_flow_id, + flow_init_parameters + ); + + // Combine processors into flow objects + local processor_array = blueprint_processors + flow_processors; + local flow_objects = flow_builder.build_flow_objects(processor_array); + local active_flows = flow_builder.merge_flow_objects(flow_objects); + + // Build interfaces for the default flow + local default_flow_interfaces = interface_builder.build_interfaces( + flow_blueprints, + default_flow_blueprint, + default_flow_id, + flow_init_parameters + ); + + // Return object with nested configuration (for backwards compatibility) + { + // Create function (for backwards compatibility) + create: function(engine) {}, + + // The actual configuration object + configuration: { + // Prompts configuration + prompt: { + "system": config_spec.prompts["system-template"], + "template-index": std.objectFieldsAll(config_spec.prompts.templates), + } + { + ["template." + template.key]: template.value + for template in std.objectKeysValuesAll(config_spec.prompts.templates) + }, + + // Tools configuration + tool: { + [tool.id]: tool + for tool in config_spec.tools + }, + + // MCP configuration + mcp: config_spec.mcp, + + // Flow blueprints reference + "flow-blueprint": flow_blueprints, + + // Interface descriptions + "interface-description": config_spec.interface_descriptions, + + // Flow instances + "flow": { + [default_flow_id]: { + "description": "Default processing flow", + "blueprint-name": default_flow_blueprint, + "interfaces": default_flow_interfaces, + "parameters": flow_init_parameters, + }, + }, + + // Active flow processors + "active-flow": active_flows, + + // Token costs and parameter types + "token-cost": config_spec.token_costs, + "parameter-type": config_spec.parameter_types, + + // Collections configuration + "collection": config_spec.collection, + + }, + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/flow-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/flow-builder.jsonnet new file mode 100644 index 00000000..0b142750 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/flow-builder.jsonnet @@ -0,0 +1,72 @@ +// Flow Builder Module +// Processes flow blueprints and builds complete flow configurations +// Handles {blueprint}, {id}, and parameter substitutions + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds blueprint-level processors with parameter substitution + // Processes the 'blueprint' section of flow blueprints + build_blueprint_processors: function(flow_blueprints, blueprint_name, parameters) + [ + [ + // Replace {blueprint} in the processor key + local key = std.strReplace(processor.key, "{blueprint}", blueprint_name); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // First replace {blueprint}, then substitute parameters + local blueprint_replaced = std.strReplace(field.value, "{blueprint}", blueprint_name); + param_processor.substitute_parameters(blueprint_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_blueprints[blueprint_name].blueprint) + ], + + // Builds flow-level processors with parameter substitution + // Processes the 'flow' section of flow blueprints + build_flow_processors: function(flow_blueprints, blueprint_name, flow_id, parameters) + [ + [ + // Replace both {blueprint} and {id} in the processor key + local key = std.strReplace( + std.strReplace(processor.key, "{blueprint}", blueprint_name), + "{id}", flow_id + ); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // Replace {blueprint} and {id}, then substitute parameters + local blueprint_replaced = std.strReplace(field.value, "{blueprint}", blueprint_name); + local id_replaced = std.strReplace(blueprint_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_blueprints[blueprint_name].flow) + ], + + // Combines blueprint and flow processors into flow objects + build_flow_objects: function(processor_array) + std.map( + function(item) { + [item[0][0]] +: { + [item[0][1]]: item[1] + } + }, + processor_array + ), + + // Merges all flow objects into a single flows_active configuration + merge_flow_objects: function(flow_objects) + std.foldr( + function(a, b) a + b, + flow_objects, + {} + ), +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/interface-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/interface-builder.jsonnet new file mode 100644 index 00000000..2143e600 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/interface-builder.jsonnet @@ -0,0 +1,30 @@ +// Interface Builder Module +// Processes flow class interfaces with parameter substitution +// Handles both string interfaces and nested object interfaces + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds interfaces for a specific flow class and instance + // Processes the 'interfaces' section of flow classes + build_interfaces: function(flow_classes, class_name, flow_id, parameters) + local interface_spec = flow_classes[class_name].interfaces; + { + [interface.key]: + if std.isString(interface.value) then + // Simple string interface - apply all substitutions + local class_replaced = std.strReplace(interface.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + else + // Complex object interface - process nested fields + { + [field.key]: + local class_replaced = std.strReplace(field.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(interface.value) + } + for interface in std.objectKeysValuesAll(interface_spec) + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/interface-descriptions.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/interface-descriptions.jsonnet new file mode 100644 index 00000000..f9dda8b0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/interface-descriptions.jsonnet @@ -0,0 +1,89 @@ +// Interface Descriptions Module +// Defines all external interfaces available in TrustGraph flows +// These are the 'endpoints' that external systems can interact with + +{ + // Document loading interfaces - for data ingestion + "document-load": { + "description": "Document loader", + "kind": "send", + "visible": true, + }, + "text-load": { + "description": "Text document loader", + "kind": "send", + "visible": true, + }, + + // Data storage interfaces - for processed data streams + "entity-contexts-load": { + "description": "Entity contexts loader", + "kind": "send", + }, + "triples-store": { + "description": "Triples loader", + "kind": "send", + }, + "graph-embeddings-store": { + "description": "Graph embeddings loader", + "kind": "send", + }, + "document-embeddings-store": { + "description": "Document embeddings loader", + "kind": "send", + }, + "objects-store": { + "description": "Object store", + "kind": "request-response", + }, + + // Query interfaces - for retrieving information + "graph-rag": { + "description": "GraphRAG service", + "kind": "request-response", + }, + "document-rag": { + "description": "ChunkRAG service", + "kind": "request-response", + }, + "triples": { + "description": "Triples query service", + "kind": "request-response", + }, + "graph-embeddings": { + "description": "Graph embeddings service", + "kind": "request-response", + }, + "document-embeddings": { + "description": "Document embeddings service", + "kind": "request-response", + }, + "objects": { + "description": "Object query service", + "kind": "request-response", + }, + + // Processing services - for text and data processing + "prompt": { + "description": "Prompt service", + "kind": "request-response", + }, + "agent": { + "description": "Agent service", + "kind": "request-response", + }, + "text-completion": { + "description": "Text completion service", + "kind": "request-response", + }, + + // Query translation services - for natural language queries + "nlp-query": { + "description": "NLP question to GraphQL service", + "kind": "request-response", + }, + "structured-query": { + "description": "Structured query service", + "kind": "request-response", + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/parameter-processor.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/parameter-processor.jsonnet new file mode 100644 index 00000000..f317ae9f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/parameter-processor.jsonnet @@ -0,0 +1,38 @@ +// Parameter Processing Module +// Handles dynamic parameter replacement in configuration values +// Replaces {parameter_name} placeholders with actual parameter values + +{ + // Applies parameter substitutions to string values + // Only processes strings - leaves other types unchanged + substitute_parameters: function(value, parameters) + if std.isString(value) then + std.foldl( + function(acc, param) + // Only do string replacement if param.value is a string + if std.isString(param.value) then + std.strReplace(acc, "{" + param.key + "}", param.value) + else + acc, // Skip replacement for non-string parameter values + std.objectKeysValuesAll(parameters), + value + ) + else + value, + + // Applies parameter substitutions to all values in an object + // Recursively processes nested objects and arrays + substitute_parameters_in_object: function(obj, parameters) + if std.isObject(obj) then + { + [key]: $.substitute_parameters_in_object(obj[key], parameters) + for key in std.objectFields(obj) + } + else if std.isArray(obj) then + [ + $.substitute_parameters_in_object(item, parameters) + for item in obj + ] + else + $.substitute_parameters(obj, parameters), +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/tools.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/tools.jsonnet new file mode 100644 index 00000000..4947279e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/tools.jsonnet @@ -0,0 +1,36 @@ +// Tools Configuration Module +// Defines all available tools that can be used by agents and flows +// Each tool specifies its interface, arguments, and behavior + +[ + // Knowledge query tool - queries the knowledge base + { + id: "knowledge-query", + name: "Knowledge query", + description: "This tool queries a knowledge base that holds information about domain-specific information. The question should be a natural language question.", + type: "knowledge-query", + collection: "default", + arguments: [ + { + name: "question", + type: "string", + description: "A simple natural language question.", + } + ] + }, + + // LLM completion tool - general purpose text completion + { + id: "llm-completion", + name: "LLM text completion", + type: "text-completion", + description: "This tool queries an LLM for non-domain-specific information. The question should be a natural language question.", + arguments: [ + { + name: "question", + type: "string", + description: "The question which should be asked of the LLM.", + } + ] + } +] \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/trustgraph-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/trustgraph-config.jsonnet new file mode 100644 index 00000000..d860ab3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/runtime-config/trustgraph-config.jsonnet @@ -0,0 +1,90 @@ +// TrustGraph Main Configuration +// Clean, modular composition of TrustGraph configuration +// Uses specialized modules for different aspects of config building + +// Import dependencies +local images = import "../values/images.jsonnet"; +local url = import "../values/url.jsonnet"; +local prompts = import "../prompts/mixtral.jsonnet"; +local default_prompts = import "../prompts/default-prompts.jsonnet"; +local token_costs = import "../values/token-costs.jsonnet"; +local flow_blueprints = import "../flows/flow-blueprints.jsonnet"; +local config_composer = import "config-composer.jsonnet"; +local interface_descriptions = import "interface-descriptions.jsonnet"; +local tools = import "tools.jsonnet"; +local temperature_params = import "../parameters/temperature-param-types.jsonnet"; +local chunking_params = import "../parameters/chunking-param-types.jsonnet"; + +// Main configuration object +local configuration = { + + // Prompt templates + prompts:: default_prompts, + + // Tool definitions + tools:: tools, + + // MCP configuration + mcp:: {}, + + // Flow classes reference + "flow-blueprints":: flow_blueprints, + + // LLM model parameters + "llm-models" +:: {}, + + // Embeddings model parameters + "embeddings-models" +:: {}, + + collections +:: { + "trustgraph:default": { + "user": "default-user", + "collection": "default", + "name": "Default Collection", + "description": "Default collection", + "tags": ["default"], + }, + }, + + // Default model and flow parameters + flow_init_parameters:: { + "llm-model": $["llm-models"].default, + "llm-rag-model": $["llm-models"].default, + "llm-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "llm-rag-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "chunk-size": std.toString($["parameter-types"]["chunk-size"].default), + "chunk-overlap": std.toString($["parameter-types"]["chunk-overlap"].default), + "embeddings-model": $["embeddings-models"].default, + }, + + // Interface descriptions for external endpoints + "interface-descriptions":: interface_descriptions, + + // Parameter type definitions + "parameter-types":: { + "llm-model": $["llm-models"], + "embeddings-model": $["embeddings-models"], + } + chunking_params + temperature_params, + + // Token costs + "token-costs":: token_costs, + + // Build the complete configuration using the composer + configuration:: config_composer.build({ + flow_blueprints: $["flow-blueprints"], + default_flow_blueprint: "everything", + default_flow_id: "default", + flow_init_parameters: $["flow_init_parameters"], + prompts: $["prompts"], + tools: $["tools"], + mcp: $["mcp"], + interface_descriptions: $["interface-descriptions"], + parameter_types: $["parameter-types"], + token_costs: $["token-costs"], + collection: $["collections"], + }), + +} + default_prompts; + +// Export the final configuration +configuration diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/cassandra.jsonnet new file mode 100644 index 00000000..05f4fd27 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/cassandra.jsonnet @@ -0,0 +1,77 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "backends/cassandra.jsonnet"; + +cassandra + { + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/falkordb.jsonnet new file mode 100644 index 00000000..b4bace2d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/falkordb.jsonnet @@ -0,0 +1,79 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local falkordb = import "backends/falkordb.jsonnet"; + +falkordb + { + + "falkordb-url":: "falkor://falkordb:6379", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/memgraph.jsonnet new file mode 100644 index 00000000..72597d72 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/memgraph.jsonnet @@ -0,0 +1,84 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local memgraph = import "backends/memgraph.jsonnet"; + +memgraph + { + + "memgraph-url":: "bolt://memgraph:7687", + "memgraph-database":: "memgraph", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/neo4j.jsonnet new file mode 100644 index 00000000..08904715 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/triple-store/neo4j.jsonnet @@ -0,0 +1,79 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local neo4j = import "backends/neo4j.jsonnet"; + +neo4j + { + + "neo4j-url":: "bolt://neo4j:7687", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/ui/workbench-ui.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/ui/workbench-ui.jsonnet new file mode 100644 index 00000000..f2048e47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/ui/workbench-ui.jsonnet @@ -0,0 +1,32 @@ +local images = import "values/images.jsonnet"; + +{ + + "workbench-ui" +: { + + create:: function(engine) + + local container = + engine.container("workbench-ui") + .with_image(images["workbench-ui"]) + .with_limits("0.1", "256M") + .with_reservations("0.1", "256M") + .with_port(8888, 8888, "ui"); + + local containerSet = engine.containers( + "workbench-ui", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8888, 8888, "ui"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/values/images.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/values/images.jsonnet new file mode 100644 index 00000000..232aeaf3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/values/images.jsonnet @@ -0,0 +1,36 @@ +local version = import "version.jsonnet"; +{ + cassandra: "docker.io/cassandra:5.0.6", +// Not working +// ceph: "quay.io/ceph/daemon:latest-reef", + neo4j: "docker.io/neo4j:2025.08.0-community-bullseye", + pulsar: "docker.io/apachepulsar/pulsar:4.1.0", + pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.4.0", + etcd: "quay.io/coreos/etcd:v3.6.4", + minio: "docker.io/minio/minio:RELEASE.2025-09-07T16-13-09Z", + garage: "docker.io/dxflrs/garage:v2.1.0", + milvus: "docker.io/milvusdb/milvus:v2.5.17", + prometheus: "docker.io/prom/prometheus:v3.8.0", + grafana: "docker.io/grafana/grafana:12.3.0", + loki: "docker.io/grafana/loki:3.6.2", + trustgraph_base: "docker.io/trustgraph/trustgraph-base:" + version, + trustgraph_flow: "docker.io/trustgraph/trustgraph-flow:" + version, + trustgraph_ocr: "docker.io/trustgraph/trustgraph-ocr:" + version, + trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version, + trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version, + trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version, + trustgraph_mcp: "docker.io/trustgraph/trustgraph-mcp:" + version, + qdrant: "docker.io/qdrant/qdrant:v1.15.4", + memgraph_mage: "docker.io/memgraph/memgraph-mage:3.5", + memgraph_lab: "docker.io/memgraph/lab:3.5.0", + falkordb: "docker.io/falkordb/falkordb:v4.12.5", + "workbench-ui": "docker.io/trustgraph/workbench-ui:1.6.4", + "ddg-mcp-server": "docker.io/trustgraph/ddg-mcp-server:0.1.0", + "tgi-service-intel-xpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-xpu", + "tgi-service-cpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-cpu", + "tgi-service-gaudi": "ghcr.io/huggingface/text-generation-inference:sha-f140440-gaudi", + "vllm-service-intel-xpu": "docker.io/intel/vllm:0.8.0-xpu", + "vllm-service-gaudi": "docker.io/trustgraph/vllm-hpu:027f5645", + "vllm-service-nvidia": "docker.io/vllm/vllm-openai:latest", + "vllm-service-intel-battlemage": "docker.io/intelanalytics/ipex-llm-serving-xpu:0.2.0-b6", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/values/token-costs.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/values/token-costs.jsonnet new file mode 100644 index 00000000..84c70373 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/values/token-costs.jsonnet @@ -0,0 +1,102 @@ +{ + "mistral.mistral-large-2407-v1:0": { + "model_name": "mistral.mistral-large-2407-v1:0", + "input_price": 0.000004, + "output_price": 0.000012 + }, + "meta.llama3-1-405b-instruct-v1:0": { + "model_name": "meta.llama3-1-405b-instruct-v1:0", + "input_price": 0.00000532, + "output_price": 0.000016 + }, + "mistral.mixtral-8x7b-instruct-v0:1": { + "model_name": "mistral.mixtral-8x7b-instruct-v0:1", + "input_price": 0.00000045, + "output_price": 0.0000007 + }, + "meta.llama3-1-70b-instruct-v1:0": { + "model_name": "meta.llama3-1-70b-instruct-v1:0", + "input_price": 0.00000099, + "output_price": 0.00000099 + }, + "meta.llama3-1-8b-instruct-v1:0": { + "model_name": "meta.llama3-1-8b-instruct-v1:0", + "input_price": 0.00000022, + "output_price": 0.00000022 + }, + "anthropic.claude-3-haiku-20240307-v1:0": { + "model_name": "anthropic.claude-3-haiku-20240307-v1:0", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "anthropic.claude-3-5-sonnet-20240620-v1:0": { + "model_name": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "cohere.command-r-plus-v1:0": { + "model_name": "cohere.command-r-plus-v1:0", + "input_price": 0.0000030, + "output_price": 0.0000150 + }, + "ollama": { + "model_name": "ollama", + "input_price": 0, + "output_price": 0 + }, + "claude-3-haiku-20240307": { + "model_name": "claude-3-haiku-20240307", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "claude-3-5-sonnet-20240620": { + "model_name": "claude-3-5-sonnet-20240620", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "claude-3-opus-20240229": { + "model_name": "claude-3-opus-20240229", + "input_price": 0.000015, + "output_price": 0.000075 + }, + "claude-3-sonnet-20240229": { + "model_name": "claude-3-sonnet-20240229", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "command-r-08-202": { + "model_name": "command-r-08-202", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "c4ai-aya-23-8b": { + "model_name": "c4ai-aya-23-8b", + "input_price": 0, + "output_price": 0 + }, + "llama.cpp": { + "model_name": "llama.cpp", + "input_price": 0, + "output_price": 0 + }, + "gpt-4o": { + "model_name": "gpt-4o", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-2024-08-06": { + "model_name": "gpt-4o-2024-08-06", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "gpt-4o-2024-05-13": { + "model_name": "gpt-4o-2024-05-13", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-mini": { + "model_name": "gpt-4o-mini", + "input_price": 0.00000015, + "output_price": 0.0000006 + } +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/values/url.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/values/url.jsonnet new file mode 100644 index 00000000..c3d4ad97 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/values/url.jsonnet @@ -0,0 +1,7 @@ +{ + pulsar: "pulsar://pulsar:6650", + pulsar_admin: "http://pulsar:8080", + milvus: "http://milvus:19530", + qdrant: "http://qdrant:6333", + object_store: "garage:3900", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/vector-store/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/vector-store/milvus.jsonnet new file mode 100644 index 00000000..213027d1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/vector-store/milvus.jsonnet @@ -0,0 +1,146 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local milvus = import "backends/milvus.jsonnet"; + +milvus + { + + "store-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/vector-store/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/vector-store/pinecone.jsonnet new file mode 100644 index 00000000..3803bdcb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/vector-store/pinecone.jsonnet @@ -0,0 +1,160 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; + +{ + + "pinecone-cloud":: "aws", + "pinecone-region":: "us-east-1", + + "store-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/vector-store/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/vector-store/qdrant.jsonnet new file mode 100644 index 00000000..06c9f0e1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/vector-store/qdrant.jsonnet @@ -0,0 +1,250 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local qdrant = import "backends/qdrant.jsonnet"; + +qdrant + { + + "store-graph-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "doc-embeddings-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "doc-embeddings-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + }, + + "store-row-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-row-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "row-embeddings-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-row-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-row-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-row-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "row-embeddings-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-row-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/zip-readme.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/zip-readme.md new file mode 100644 index 00000000..0b117792 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.0/zip-readme.md @@ -0,0 +1,28 @@ + +Note! this is a subset of possible configurations, to generate your own +launch config use the config util... + +- Production: https://config-ui.demo.trustgraph.ai +- Early release: https://dev.config-ui.demo.trustgraph.ai + +The config util auto-generates deployment instructions for your +configuration, so that's the recommended way to deploy. + +---------------------------------------------------------------------------- + +These are launch configurations for TrustGraph. See https://trustgraph.ai for +the quickstart using docker compose. + +Hint for Linux: There are files here which get mounted as volumes inside +Docker Compose containers. This may trigger SELinux rules on your system, to +permit access insider the containers, use a command like this... + +chcon -Rt svirt_sandbox_file_t grafana/ prometheus/ + +The file vertexai/private.json is a placeholder for real GCP credentials if +you are using the VertexAI LLM. If you're using that in Docker Compose, +replace with your real credentials, and don't forget to permit access if you +are using Linux: + +chcon -Rt svirt_sandbox_file_t vertexai/ + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/README.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/README.md new file mode 100644 index 00000000..23039e9a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/README.md @@ -0,0 +1,125 @@ + +# TrustGraph template generation + +There are two utilities here: + +- `generate`: Generates a single Docker Compose launch configuration + based on configuration you provide. +- `generate-all`: Generates the release bundle for releases. You won't + need to use this unless you are managing releases. + +## `generate-all` + +Previously, this generates a full set of all vector DB / triple store / LLM +combinations, and put them in a single ZIP file. But this got out of +hand, so at the time of writing, this generates a single configuraton +using Qdrant vector DB, Ollama LLM support and Cassandra for a triple store. + +The combinations are contained withing the code, it takes two arguments: +- output ZIP file (is over-written) +- TrustGraph version number + +``` +templates/generate-all output.zip 0.18.11 +``` + +## `generate` + +This utility takes a configuration file describing the components to bundle, +and outputs a Docker Compose YAML file. + +### Input configuration + +The input configuration is a JSON file, an array of components to pull into +the configuration. For each component, there is a name and a (possibly empty) +object describing addtional parameters for that component. + +Example: + +``` +[ + { + "name": "cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "qdrant", + "parameters": {} + }, + { + "name": "embeddings-hf", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph", + "parameters": {} + }, + { + "name": "googleaistudio", + "parameters": { + "googleaistudio-temperature": 0.3, + "googleaistudio-max-output-tokens": 2048, + "googleaistudio-model": "gemini-1.5-pro-002" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 1000, + "chunk-overlap": 50 + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "agent-manager-react", + "parameters": {} + } +] +``` + +If you want to make your own configuration you could try changing the +configuration above: +- Components which are essential: pulsar, trustgraph, graph-rag, grafana, + agent-manager-react +- You need a triple store, one of: cassandra, memgraph, falkordb, neo4j +- You need a vector store, one of: qdrant, pinecone +- You need an LLM, one of: azure, azure-openai, bedrock, claude, cohere, + llamafile, ollama, openai, vertexai. +- You need an embeddings implementation, one of: embeddings-hf, + embeddings-ollama +- Optionally add the Workbench tool: workbench-ui + +Components have over-ridable parameters, look in the component definition +in `templates/components/` to see what you can override. + +### Invocation + +Two parameters: +- The output ZIP file +- The version number + +The configuration file described above is provided on standard input + +``` +templates/generate out.zip 0.18.9 < config.json +``` + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/cassandra.jsonnet new file mode 100644 index 00000000..60f262d3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/cassandra.jsonnet @@ -0,0 +1,50 @@ +local images = import "values/images.jsonnet"; + +{ + + "cassandra" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1000M", + "memory-reservation":: "1000M", + "heap":: "300M", + + create:: function(engine) + + // Capture memory settings into locals + local memLimit = self["memory-limit"]; + local memReserv = self["memory-reservation"]; + local heap = self["heap"]; + + local vol = engine.volume("cassandra").with_size("20G"); + + local container = + engine.container("cassandra") + .with_image(images.cassandra) + .with_environment({ + JVM_OPTS: "-Xms%s -Xmx%s -Dcassandra.skip_wait_for_gossip_to_settle=0" % [ + heap, heap, + ], + }) + .with_limits("1.0", memLimit) + .with_reservations("0.5", memReserv) + .with_port(9042, 9042, "cassandra") + .with_volume_mount(vol, "/var/lib/cassandra"); + + local containerSet = engine.containers( + "cassandra", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9042, 9042, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/ceph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/ceph.jsonnet new file mode 100644 index 00000000..42406a95 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/ceph.jsonnet @@ -0,0 +1,446 @@ + +// +// This majorly does not work +// + +// This configuration fails primarily because it tries to treat Ceph +// like a stateless web app. You are currently pointing mon host to a +// generic service name (ceph-mon), but you aren't telling the Monitor +// process to assume that service identity. + +// To make this work with the "Service Name" approach across any +// engine, you need to fix the binding logic and the messenger protocol. + +// 1. The ceph.conf Fix + +// Need to enable Messenger v2 (modern) and tell the cluster to use +// the service names for its initial quorum. + +//Change this section: +// Ini, TOML + +// [global] +// fsid = %s +// mon initial members = mon0 +// mon host = ceph-mon:6789 +// ... + +// To this: +// Ini, TOML + +// [global] +// fsid = %s +// # Use the actual service names as members +// mon initial members = ceph-mon +// # Explicitly use the Service name (VIP) +// mon host = ceph-mon +// # Force modern protocol +// ms_bind_msgr2 = true +// ms_bind_msgr1 = true + +// 2. The MON Environment & Command Fix + +// This is the most critical part. Your current config sets MON_IP: +// "0.0.0.0". This causes the MON to bind to the Pod IP, which breaks +// when the Pod restarts. You must force it to bind to the Service IP. + +// Update mon_env: +// Code snippet + +// local mon_env = cluster_env + { +// CEPH_DAEMON: "MON", +// MON_NAME: "mon0", +// # Remove MON_IP: "0.0.0.0" +// # Add these: +// MON_ADDR: "ceph-mon", // This says to resolve the service name +// }; + +// Update mon_container command: You are currently wiping the MON data +// on every start (rm -rf /var/lib/ceph/mon/*). Stop doing that. If you +// wipe the data, you lose the cluster state and the OSDs will refuse to +// talk to the "new" MON. + +// Code snippet + +// .with_command([ +// "bash", "-c", +// # 1. Resolve the Service IP at runtime +// "export MON_IP=$(getent hosts ceph-mon | awk '{ print $1 }'); " + +// inject_mon_config + +// # 2. Start the daemon telling it its PUBLIC address is the Service VIP +// "exec /opt/ceph-container/bin/entrypoint.sh" +// ]) + +// 3. Why your current config "Majorly does not work" + +// The "Wipe" Logic: By running rm -rf /var/lib/ceph/mon/* in the +// MON container, you are creating a "New Cluster" every time the +// container starts. Since the OSDs store the fsid and cluster +// secrets, they will reject the "new" MON. + +// DNS Race Condition: Your OSD/MGR/RGW containers wait for +// ceph-mon DNS, which is good. However, if ceph-mon resolves to a +// Round Robin IP (multiple pods) rather than a stable ClusterIP, the +// connection will be flaky. + +// Messenger Protocol: Without ms_bind_msgr2, Ceph defaults to the +// old v1 protocol which is much more sensitive to NAT/Container IP +// mismatches. + +// SUMMARY + +// Component: ceph.conf +// Change: Add ms_bind_msgr2 = true +// Why: Supports modern container networking better. + +// Component: MON Start +// Change: Remove rm -rf +// Why: Ceph MONs must keep their database to maintain the cluster. + +// Component: MON Address +// Change: Use getent hosts ceph-mon +// Why: Forces the MON to advertise the Service VIP instead of its own Pod IP. + +// Component: MON Keyring +// Change: Ensure /etc/ceph/ceph.mon.keyring exists +// Why: MONs need their specific key to start. + +local images = import "values/images.jsonnet"; + +{ + with:: function(key, value) + self + { + ["ceph-" + key]:: value, + }, + + // Ceph credentials and cluster settings + "ceph-access-key":: "object-user", + "ceph-secret-key":: "object-password", + "ceph-cluster-id":: "ceph", + "ceph-fsid":: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + + // Pool redundancy settings + // size: 2 = two replicas for fault tolerance + // min_size: 1 = allow degraded I/O if one OSD is down (prevents cluster freeze) + "ceph-pool-size":: "2", + "ceph-pool-min-size":: "1", + + ceph +: { + create:: function(engine) + // Pre-Shared Cryptographic Material - Config-as-Code Approach + // These keys are generated once and distributed to all daemons + // This ensures cryptographic consistency across the shared-nothing architecture + local admin_key = "AQBpxSBlAAAAABAAU99V6D8vS7Uu9y1S8W0iBg=="; + local mon_key = "AQBpxSBlAAAAABAAn7pL/pG9oT+X6vO7V1S6bg=="; + + // Ceph configuration file - rendered from Jsonnet variables + local ceph_conf = ||| + [global] + fsid = %s + mon initial members = mon0 + mon host = ceph-mon:6789 + public network = 0.0.0.0/0 + cluster network = 0.0.0.0/0 + osd pool default size = %s + osd pool default min size = %s + osd crush chooseleaf type = 0 + auth cluster required = cephx + auth service required = cephx + auth client required = cephx + ||| % [$["ceph-fsid"], $["ceph-pool-size"], $["ceph-pool-min-size"]]; + + // Admin keyring - distributed to all daemons + local admin_keyring = ||| + [client.admin] + key = %s + caps mds = "allow *" + caps mgr = "allow *" + caps mon = "allow *" + caps osd = "allow *" + ||| % [admin_key]; + + // Monitor keyring - used by MON for cluster operations + local mon_keyring = ||| + [mon.] + key = %s + caps mon = "allow *" + ||| % [mon_key]; + + // Config injection command - writes files before entrypoint + local inject_config = "printf '%s' > /etc/ceph/ceph.conf; printf '%s' > /etc/ceph/ceph.client.admin.keyring; " % [ceph_conf, admin_keyring]; + local inject_mon_config = inject_config + ("printf '%s' > /etc/ceph/ceph.mon.keyring; " % [mon_keyring]); + + // Data volumes - sized appropriately for production workloads + local vol_mon = engine.volume("ceph-mon").with_size("20G"); + local vol_mgr = engine.volume("ceph-mgr").with_size("20G"); + local vol_osd = engine.volume("ceph-osd").with_size("100G"); + local vol_rgw = engine.volume("ceph-rgw").with_size("20G"); + + // Isolated config volumes per daemon (ReadWriteOnce compatible) + // Each daemon gets its own non-shared config volume to support + // multi-node scheduling in K8s and other orchestrators + local vol_mon_config = engine.volume("ceph-mon-config").with_size("500M"); + local vol_mgr_config = engine.volume("ceph-mgr-config").with_size("500M"); + local vol_osd_config = engine.volume("ceph-osd-config").with_size("500M"); + local vol_rgw_config = engine.volume("ceph-rgw-config").with_size("500M"); + local vol_init_config = engine.volume("ceph-init-config").with_size("500M"); + + // Simplified cluster environment - Config-as-Code model + // No fetch logic needed - config is injected before entrypoint runs + local cluster_env = { + CLUSTER: $["ceph-cluster-id"], + FSID: $["ceph-fsid"], + KV_TYPE: "none", // No external coordination + }; + + // MON-specific environment + // Config-as-Code: MON uses injected config files, not fetch logic + // + // CRITICAL: MON_DATA_AVAIL="0" forces fresh cluster bootstrap + // The ceph/daemon entrypoint script (variables_stack.sh) uses this as a gate: + // - MON_DATA_AVAIL="0" -> run mkfs, create new cluster with our FSID + // - MON_DATA_AVAIL="1" -> attempt to join existing cluster (infinite probe loop) + // + // Network configuration for monmap generation + local mon_env = cluster_env + { + CEPH_DAEMON: "MON", + MON_NAME: "mon0", + MON_PORT: "6789", + MON_DATA_AVAIL: "0", + MON_IP: "0.0.0.0", + NETWORK_AUTO_DETECT: "4", + CEPH_PUBLIC_NETWORK: "0.0.0.0/0", + }; + + // Simplified daemon environments - Config-as-Code model + // All daemons receive config via injection, not fetch from MON + // This eliminates "static mode" errors and networking complexity + + // MGR-specific environment + local mgr_env = cluster_env + { + CEPH_DAEMON: "MGR", + MGR_NAME: "mgr0", + }; + + // OSD-specific environment + local osd_env = cluster_env + { + CEPH_DAEMON: "OSD", + OSD_TYPE: "directory", + }; + + // RGW-specific environment + local rgw_env = cluster_env + { + CEPH_DAEMON: "RGW", + RGW_NAME: "rgw0", + RGW_FRONTEND_PORT: "7480", + }; + + // MON (Monitor) container - cluster state and quorum + // Config-as-Code: Injects pre-shared keys before entrypoint + // CRITICAL: Wipes /var/lib/ceph/mon/* on every start to force fresh bootstrap + // This ensures MON always uses our FSID and doesn't inherit stale cluster state + local mon_container = + engine.container("ceph-mon") + .with_image(images.ceph) + .with_environment(mon_env) + .with_command([ + "bash", "-c", + "rm -rf /var/lib/ceph/mon/*; " + + inject_mon_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(6789, 6789, "mon") + .with_port(3300, 3300, "mon-msgr2") + .with_volume_mount(vol_mon, "/var/lib/ceph/mon") + .with_volume_mount(vol_mon_config, "/etc/ceph"); + + // MGR (Manager) container - cluster management and dashboard + // Config-as-Code: Uses injected config files with pre-shared keys + // DNS wait ensures MON is available before MGR connects + local mgr_container = + engine.container("ceph-mgr") + .with_image(images.ceph) + .with_environment(mgr_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(7000, 7000, "mgr") + .with_port(8443, 8443, "dashboard") + .with_port(9283, 9283, "prometheus") + .with_volume_mount(vol_mgr, "/var/lib/ceph/mgr") + .with_volume_mount(vol_mgr_config, "/etc/ceph"); + + // OSD (Object Storage Daemon) - actual data storage + // Config-as-Code: Uses injected config files with pre-shared keys + // Increased resources to prevent OOM during recovery operations + // DNS wait ensures MON is available before OSD connects + local osd_container = + engine.container("ceph-osd") + .with_image(images.ceph) + .with_environment(osd_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("2.0", "4096M") + .with_reservations("1.0", "2048M") + .with_port(6800, 6800, "osd") + .with_volume_mount(vol_osd, "/var/lib/ceph/osd") + .with_volume_mount(vol_osd_config, "/etc/ceph"); + + // RGW (RADOS Gateway) - S3 API endpoint + // Config-as-Code: Uses injected config files with pre-shared keys + // DNS wait ensures MON is available before RGW connects + local rgw_container = + engine.container("ceph-rgw") + .with_image(images.ceph) + .with_environment(rgw_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(7480, 7480, "s3") + .with_volume_mount(vol_rgw, "/var/lib/ceph/radosgw") + .with_volume_mount(vol_rgw_config, "/etc/ceph"); + + // Init container - one-time S3 user provisioning + // IMPORTANT: This container exits with code 0 after completion + // Orchestrator must NOT restart it (use K8s Job or Compose restart: "no") + // Config-as-Code: Uses injected config to run radosgw-admin commands + local init_container = + engine.container("ceph-init") + .with_image(images.ceph) + .with_environment({ + CLUSTER: $["ceph-cluster-id"], + FSID: $["ceph-fsid"], + KV_TYPE: "none", + RGW_ACCESS_KEY: $["ceph-access-key"], + RGW_SECRET_KEY: $["ceph-secret-key"], + }) + .with_limits("0.5", "512M") + .with_reservations("0.25", "256M") + .with_volume_mount(vol_init_config, "/etc/ceph") + .with_command([ + "bash", "-c", + inject_config + ||| + set -e + + # Wait for cluster health + echo "Waiting for Ceph cluster to be healthy..." + MAX_ATTEMPTS=60 + ATTEMPT=0 + until ceph --cluster ${CLUSTER} health 2>/dev/null | grep -q "HEALTH_OK\|HEALTH_WARN"; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: Cluster failed to become healthy after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: Cluster not ready, retrying in 5s..." + sleep 5 + done + echo "Cluster is healthy." + + # Wait for RGW availability + echo "Waiting for RGW to be ready..." + ATTEMPT=0 + until curl -sf http://ceph-rgw:7480 >/dev/null 2>&1; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: RGW failed to become ready after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: RGW not ready, retrying in 5s..." + sleep 5 + done + echo "RGW is ready." + + # Idempotent S3 user creation + echo "Provisioning S3 user: ${RGW_ACCESS_KEY}" + if radosgw-admin --cluster ${CLUSTER} user info --uid="${RGW_ACCESS_KEY}" >/dev/null 2>&1; then + echo "User ${RGW_ACCESS_KEY} already exists, skipping creation." + else + echo "Creating new S3 user: ${RGW_ACCESS_KEY}" + radosgw-admin --cluster ${CLUSTER} user create \ + --uid="${RGW_ACCESS_KEY}" \ + --display-name="Object Storage User" \ + --access-key="${RGW_ACCESS_KEY}" \ + --secret-key="${RGW_SECRET_KEY}" + echo "S3 user created successfully." + fi + + echo "Initialization complete. Exiting." + exit 0 + |||, + ]); + + // Container sets - each daemon gets its own for K8s node distribution + local mon_containerSet = engine.containers("ceph-mon", [mon_container]); + local mgr_containerSet = engine.containers("ceph-mgr", [mgr_container]); + local osd_containerSet = engine.containers("ceph-osd", [osd_container]); + local rgw_containerSet = engine.containers("ceph-rgw", [rgw_container]); + local init_containerSet = engine.containers("ceph-init", [init_container]); + + // Services - expose daemon ports for inter-daemon communication + local mon_service = + engine.service(mon_containerSet) + .with_port(6789, 6789, "mon") + .with_port(3300, 3300, "mon-msgr2"); + + local mgr_service = + engine.service(mgr_containerSet) + .with_port(7000, 7000, "mgr") + .with_port(8443, 8443, "dashboard") + .with_port(9283, 9283, "prometheus"); + + local osd_service = + engine.service(osd_containerSet) + .with_port(6800, 6800, "osd"); + + local rgw_service = + engine.service(rgw_containerSet) + .with_port(7480, 7480, "s3"); + + engine.resources([ + + // Data volumes + vol_mon, + vol_mgr, + vol_osd, + vol_rgw, + + // Config volumes (isolated, no sharing) + vol_mon_config, + vol_mgr_config, + vol_osd_config, + vol_rgw_config, + vol_init_config, + + // Container sets + mon_containerSet, + mgr_containerSet, + osd_containerSet, + rgw_containerSet, + init_containerSet, + + // Services + mon_service, + mgr_service, + osd_service, + rgw_service, + + ]) + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/falkordb.jsonnet new file mode 100644 index 00000000..1d4176d8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/falkordb.jsonnet @@ -0,0 +1,38 @@ +local images = import "values/images.jsonnet"; + +{ + + "falkordb" +: { + + create:: function(engine) + + local vol = engine.volume("falkordb").with_size("20G"); + + local container = + engine.container("falkordb") + .with_image(images.falkordb) + .with_limits("1.0", "768M") + .with_reservations("0.5", "768M") + .with_port(6379, 6379, "api") + .with_port(3010, 3000, "ui") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "falkordb", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6379, 6379, "api") + .with_port(3010, 3010, "ui"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/garage.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/garage.jsonnet new file mode 100644 index 00000000..9d339bfb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/garage.jsonnet @@ -0,0 +1,249 @@ +local images = import "values/images.jsonnet"; + +{ + + garage +: { + + // Garage S3 credentials - these are the actual access key ID and secret key + // Access Key ID must be in format: GK + 24 hex characters (12 bytes) + // Secret Key must be 64 hex characters (32 bytes) + // For production, generate secure random values and override these defaults + "access-key":: "GK000000000000000000000001", + "secret-key":: "b171f00be9be4c32c734f4c05fe64c527a8ab5eb823b376cfa8c2531f70fc427", + "rpc-secret":: "bbba746a9e289bad64a9e7a36a4299dac8d6e0b8cc2a6c2937fe756df4492008", + // For a production system, override this value + "admin-token":: "batts-rockhearted-unpartially", + region:: "garage", + "replication-factor":: "1", // Set to 1 for single-node, 3 for production + + // Storage volume sizes + "meta-size":: "2G", // Metadata volume size + "data-size":: "5G", // Data volume size (also used for cluster layout capacity) + + create:: function(engine) + + local accessKey = self["access-key"]; + local secretKey = self["secret-key"]; + local rpcSecret = self["rpc-secret"]; + local adminToken = self["admin-token"]; + local region = self.region; + local replicationFactor = self["replication-factor"]; + local metaSize = self["meta-size"]; + local dataSize = self["data-size"]; + + // Garage daemon configuration file - TOML format + local garage_conf = ||| + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + + db_engine = "lmdb" + + replication_factor = %s + + compression_level = 1 + + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "[::]:3901" + rpc_secret = "%s" + + [s3_api] + s3_region = "%s" + api_bind_addr = "[::]:3900" + root_domain = ".s3.garage.local" + + [s3_web] + bind_addr = "[::]:3902" + root_domain = ".web.garage.local" + index = "index.html" + + [k2v_api] + api_bind_addr = "[::]:3904" + + [admin] + api_bind_addr = "[::]:3903" + admin_token = "%s" + ||| % [replicationFactor, rpcSecret, region, adminToken]; + + // Config volume - contains the rendered garage.toml + local cfgVol = engine.configVolume( + "garage-cfg", "garage", + { + "garage.toml": garage_conf, + } + ); + + // Volumes - Garage stores metadata and data separately + local vol_meta = engine.volume("garage-meta").with_size(metaSize); + local vol_data = engine.volume("garage-data").with_size(dataSize); + + // Main Garage daemon container + local garage_container = + engine.container("garage") + .with_image(images.garage) + .with_command([ + "/garage", "-c", "/etc/garage/garage.toml", "server" + ]) + .with_environment({ + RUST_LOG: "garage=info", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3900, 3900, "s3-api") + .with_port(3901, 3901, "rpc") + .with_port(3902, 3902, "web") + .with_port(3903, 3903, "admin") + .with_port(3904, 3904, "k2v") + .with_volume_mount(cfgVol, "/etc/garage/") + .with_volume_mount(vol_meta, "/var/lib/garage/meta") + .with_volume_mount(vol_data, "/var/lib/garage/data"); + + // Init container - configures cluster layout and creates S3 credentials + // IMPORTANT: This container exits with code 0 after completion + // Orchestrator must NOT restart it (use K8s Job or Compose restart: "no") + // Uses Alpine base image since garage container has no shell + local init_container = + engine.container("garage-init") + .with_image("docker.io/alpine:3.23.2") + .with_environment({ + GARAGE_ACCESS_KEY: accessKey, + GARAGE_SECRET_KEY: secretKey, + GARAGE_REGION: region, + GARAGE_ADMIN_TOKEN: adminToken, + GARAGE_RPC_SECRET: rpcSecret, + GARAGE_DATA_SIZE: dataSize, + }) + .with_limits("0.5", "256M") + .with_reservations("0.25", "128M") + .with_volume_mount(cfgVol, "/etc/garage/") + .with_command([ + "sh", "-c", ||| + set -e + + # Install required tools + echo "Installing curl, jq and downloading garage CLI..." + apk add --no-cache curl jq + + # Download garage binary (v2.1.0) for remote management + curl -fsSL "https://garagehq.deuxfleurs.fr/_releases/v2.1.0/x86_64-unknown-linux-musl/garage" \ + -o /usr/local/bin/garage + chmod +x /usr/local/bin/garage + + echo "Waiting for Garage daemon to be ready..." + MAX_ATTEMPTS=60 + ATTEMPT=0 + # Wait for /health to respond (even 503 is fine - means daemon is up) + until curl -s http://garage:3903/health >/dev/null 2>&1; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: Garage failed to become ready after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: Garage not ready, retrying in 2s..." + sleep 2 + done + echo "Garage daemon is ready." + + # Get the node ID via v2 Admin API + echo "Getting Garage node ID via Admin API..." + curl -s -H "Authorization: Bearer ${GARAGE_ADMIN_TOKEN}" \ + "http://garage:3903/v2/GetNodeInfo?node=self" > /tmp/garage-node-info.json + + # Extract node ID from response (the key in success map is the node ID) + NODE_ID=$(jq -r '.success | to_entries[0].value.nodeId' /tmp/garage-node-info.json) + echo "Node ID: ${NODE_ID}" + + if [ -z "$NODE_ID" ] || [ "$NODE_ID" = "null" ]; then + echo "ERROR: Failed to retrieve node ID" + exit 1 + fi + + # ===== LAYOUT MANAGEMENT VIA REMOTE RPC ===== + # Use garage CLI with -h and -s flags to connect remotely + # -h requires format: @: + # -s provides the RPC secret + + # Construct full RPC identifier + RPC_HOST="${NODE_ID}@garage:3901" + echo "RPC Host: ${RPC_HOST}" + + # Check current layout to see if node is already assigned (idempotent) + echo "Checking current cluster layout..." + LAYOUT_OUTPUT=$(garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" layout show 2>&1) + + # Check if node already has a role assigned + # Layout output shows abbreviated node ID (first 16 chars) + NODE_ID_SHORT="${NODE_ID:0:16}" + if echo "$LAYOUT_OUTPUT" | grep -q "$NODE_ID_SHORT"; then + echo "Node ${NODE_ID_SHORT}... already assigned in layout, skipping." + else + echo "Assigning node to cluster layout..." + # Assign node to zone dc1 with configured capacity + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + layout assign ${NODE_ID} -z dc1 -c ${GARAGE_DATA_SIZE} + + echo "Applying layout configuration..." + # Get current staged version and apply + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + layout apply --version 1 + + echo "Layout configured successfully." + # Wait for layout to stabilize + sleep 5 + fi + + # ===== KEY MANAGEMENT VIA REMOTE RPC ===== + + # Check if key already exists (idempotent) + # GARAGE_ACCESS_KEY is already a valid Garage Key ID (GK + 24 hex chars) + if garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" key info "${GARAGE_ACCESS_KEY}" >/dev/null 2>&1; then + echo "Access key ${GARAGE_ACCESS_KEY} already exists, skipping creation." + else + echo "Importing S3 access key: ${GARAGE_ACCESS_KEY}" + + # Import key with the provided credentials (both already in valid Garage format) + # GARAGE_ACCESS_KEY = Key ID (GK + 24 hex chars) + # GARAGE_SECRET_KEY = Secret (64 hex chars) + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + key import "${GARAGE_ACCESS_KEY}" "${GARAGE_SECRET_KEY}" --yes + + echo "Access key imported successfully." + fi + + # Grant permissions to the key + echo "Granting create-bucket permission to key..." + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + key allow --create-bucket "${GARAGE_ACCESS_KEY}" + + echo "" + echo "Garage initialization complete!" + echo "S3 Endpoint: http://garage:3900" + echo "Region: ${GARAGE_REGION}" + exit 0 + |||, + ]); + + // Container sets + local garage_containerSet = engine.containers("garage", [garage_container]); + local init_containerSet = engine.containers("garage-init", [init_container]); + + // Service - expose Garage ports + local garage_service = + engine.service(garage_containerSet) + .with_port(3900, 3900, "s3-api") + .with_port(3901, 3901, "rpc") + .with_port(3902, 3902, "web") + .with_port(3903, 3903, "admin") + .with_port(3904, 3904, "k2v"); + + engine.resources([ + cfgVol, + vol_meta, + vol_data, + garage_containerSet, + init_containerSet, + garage_service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/memgraph.jsonnet new file mode 100644 index 00000000..eeed2e4e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/memgraph.jsonnet @@ -0,0 +1,70 @@ +local images = import "values/images.jsonnet"; + +{ + + "memgraph" +: { + + create:: function(engine) + + local vol = engine.volume("memgraph").with_size("20G"); + + local container = + engine.container("memgraph") + .with_image(images.memgraph_mage) + .with_environment({ + MEMGRAPH: "--storage-properties-on-edges=true --storage-enable-edges-metadata=true" + }) + .with_limits("1.0", "1000M") + .with_reservations("0.5", "1000M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/var/lib/memgraph"); + + local containerSet = engine.containers( + "memgraph", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + "memgraph-lab" +: { + + create:: function(engine) + + local container = + engine.container("lab") + .with_image(images.memgraph_lab) + .with_environment({ + QUICK_CONNECT_MG_HOST: "memgraph", + QUICK_CONNECT_MG_PORT: "7687", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3010, 3000, "http"); + + local containerSet = engine.containers( + "lab", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3010, 3010, "http"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/milvus.jsonnet new file mode 100644 index 00000000..5bed3820 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/milvus.jsonnet @@ -0,0 +1,89 @@ +local images = import "values/images.jsonnet"; +local minio = import "backends/minio.jsonnet"; + +minio { + + etcd +: { + + create:: function(engine) + + local vol = engine.volume("etcd").with_size("20G"); + + local container = + engine.container("etcd") + .with_image(images.etcd) + .with_command([ + "etcd", + "-advertise-client-urls=http://127.0.0.1:2379", + "-listen-client-urls", + "http://0.0.0.0:2379", + "--data-dir", + "/etcd", + ]) + .with_environment({ + ETCD_AUTO_COMPACTION_MODE: "revision", + ETCD_AUTO_COMPACTION_RETENTION: "1000", + ETCD_QUOTA_BACKEND_BYTES: "4294967296", + ETCD_SNAPSHOT_COUNT: "50000" + }) + .with_limits("1.0", "128M") + .with_reservations("0.25", "128M") + .with_port(2379, 2379, "api") + .with_volume_mount(vol, "/etcd"); + + local containerSet = engine.containers( + "etcd", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(2379, 2379, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + milvus +: { + + create:: function(engine) + + local vol = engine.volume("milvus").with_size("20G"); + + local container = + engine.container("milvus") + .with_image(images.milvus) + .with_command([ + "milvus", "run", "standalone" + ]) + .with_environment({ + ETCD_ENDPOINTS: "etcd:2379", + MINIO_ADDRESS: "minio:9000", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2") + .with_volume_mount(vol, "/var/lib/milvus"); + + local containerSet = engine.containers( + "milvus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/minio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/minio.jsonnet new file mode 100644 index 00000000..b38bb81f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/minio.jsonnet @@ -0,0 +1,48 @@ +local images = import "values/images.jsonnet"; + +{ + + minio +: { + + create:: function(engine) + + local vol = engine.volume("minio-data").with_size("20G"); + + local container = + engine.container("minio") + .with_image(images.minio) + .with_command([ + "minio", + "server", + "/minio_data", + "--console-address", + ":9001", + ]) + .with_environment({ + MINIO_ROOT_USER: "minioadmin", + MINIO_ROOT_PASSWORD: "minioadmin", + }) + .with_limits("0.5", "128M") + .with_reservations("0.25", "128M") + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console") + .with_volume_mount(vol, "/minio_data"); + + local containerSet = engine.containers( + "minio", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/neo4j.jsonnet new file mode 100644 index 00000000..46c61e0f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/neo4j.jsonnet @@ -0,0 +1,46 @@ +local images = import "values/images.jsonnet"; + +{ + + "neo4j" +: { + + create:: function(engine) + + local vol = engine.volume("neo4j").with_size("20G"); + + local container = + engine.container("neo4j") + .with_image(images.neo4j) + .with_environment({ + NEO4J_AUTH: "neo4j/password", + NEO4J_server_memory_pagecache_size: "512m", + NEO4J_server_memory_heap_max__size: "512m", + // NEO4J_server_bolt_listen__address: "0.0.0.0:7687", + // NEO4J_server_default__listen__address: "0.0.0.0", + // NEO4J_server_http_listen__address: "0.0.0.0:7474", + }) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1536M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "neo4j", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/pinecone.jsonnet new file mode 100644 index 00000000..2bef70fb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/pinecone.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "pinecone" +: { + + create:: function(engine) + + local container = + engine.container("pinecone") + .with_image(images.pinecone) + .with_environment({ + + PORT: "5080", + + INDEX_TYPE: "serverless", + + // Dimension is fixed, 384 is right for miniLM + // sentence embedding + DIMENSION: 384, + + METRIC: "cosine" + + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(5080, 5080, "api"); + + local containerSet = engine.containers( + "pinecone", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(5080, 5080, "api"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/qdrant.jsonnet new file mode 100644 index 00000000..2e6761f8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/backends/qdrant.jsonnet @@ -0,0 +1,65 @@ +local images = import "values/images.jsonnet"; + +{ + + "qdrant" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1024M", + "memory-reservation":: "1024M", + + // Mmap settings for low-memory mode (trades latency for memory) + // Set to null to disable, or string value to enable + "memmap-threshold-kb":: null, + "on-disk-payload":: null, + + create:: function(engine) + + // Capture memory settings into locals + local memLimit = self["memory-limit"]; + local memReserv = self["memory-reservation"]; + local mmapThreshold = self["memmap-threshold-kb"]; + local onDiskPayload = self["on-disk-payload"]; + + local vol = engine.volume("qdrant").with_size("20G"); + + // Build environment with optional mmap settings + local baseEnv = {}; + local env = baseEnv + + (if mmapThreshold != null then { + QDRANT__STORAGE__MEMMAP_THRESHOLD_KB: mmapThreshold, + } else {}) + + (if onDiskPayload != null then { + QDRANT__STORAGE__ON_DISK_PAYLOAD: onDiskPayload, + } else {}); + + local container = + engine.container("qdrant") + .with_image(images.qdrant) + .with_limits("1.0", memLimit) + .with_reservations("0.5", memReserv) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2") + .with_volume_mount(vol, "/qdrant/storage") + + (if std.length(env) > 0 then { + environment+: env, + } else {}); + + local containerSet = engine.containers( + "qdrant", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/components.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/components.jsonnet new file mode 100644 index 00000000..f6157326 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/components.jsonnet @@ -0,0 +1,121 @@ +{ + + // Essentials + "trustgraph-base": import "core/trustgraph.jsonnet", + "rev-gateway": import "core/rev-gateway.jsonnet", + "pulsar": import "pulsar/pulsar.jsonnet", + + // LLMs + "azure": import "llm/azure.jsonnet", + "azure-openai": import "llm/azure-openai.jsonnet", + "bedrock": import "llm/bedrock.jsonnet", + "claude": import "llm/claude.jsonnet", + "cohere": import "llm/cohere.jsonnet", + "googleaistudio": import "llm/googleaistudio.jsonnet", + "llamafile": import "llm/llamafile.jsonnet", + "lmstudio": import "llm/lmstudio.jsonnet", + "mistral": import "llm/mistral.jsonnet", + "ollama": import "llm/ollama.jsonnet", + "openai": import "llm/openai.jsonnet", + "vertexai": import "llm/vertexai.jsonnet", + "tgi": import "llm/tgi.jsonnet", + "vllm": import "llm/vllm.jsonnet", + + // Embeddings + "embeddings-ollama": import "embeddings/embeddings-ollama.jsonnet", + "embeddings-hf": import "embeddings/embeddings-hf.jsonnet", + "embeddings-fastembed": import "embeddings/embeddings-fastembed.jsonnet", + + // OCR options + "ocr": import "ocr/ocr.jsonnet", + "mistral-ocr": import "ocr/mistral-ocr.jsonnet", + + // Vector stores + "vector-store-milvus": import "vector-store/milvus.jsonnet", + "vector-store-qdrant": import "vector-store/qdrant.jsonnet", + "vector-store-pinecone": import "vector-store/pinecone.jsonnet", + + // Triples stores + "triple-store-cassandra": import "triple-store/cassandra.jsonnet", + "triple-store-neo4j": import "triple-store/neo4j.jsonnet", + "triple-store-falkordb": import "triple-store/falkordb.jsonnet", + "triple-store-memgraph": import "triple-store/memgraph.jsonnet", + + // Row stores + "row-store-cassandra": import "row-store/cassandra.jsonnet", + + // Observability support + "grafana": import "monitoring/grafana.jsonnet", + "loki": import "monitoring/loki.jsonnet", + + // Pulsar manager is a UI for Pulsar. Uses a LOT of memory + "pulsar-manager": import "pulsar/pulsar-manager.jsonnet", + + "override-recursive-chunker": import "core/chunker-recursive.jsonnet", + + // The prompt manager + "prompt-overrides": import "core/prompt-overrides.jsonnet", + + // Extra MCP services + "ddg-mcp-server": import "mcp/ddg-mcp-server.jsonnet", + + // Does nothing. But, can be a hack to overwrite parameters + "null": {}, + + // Passthrough: returns parameters directly, preserving +: merge syntax + // Also supports JSON-safe routing with prefixed parameters like "cassandra-heap" + "override": { + local route = function(target) + function(prefix, k, v) + local suffix = std.substr(k, std.length(prefix), std.length(k) - std.length(prefix)); + { [target] +: { [suffix]:: v } }, + + local routes = { + "cassandra-": route("cassandra"), + "pulsar-": route("pulsar"), + "qdrant-": route("qdrant"), + "api-gateway-": route("api-gateway"), + "librarian-": route("librarian"), + }, + + with_params:: function(pars) + std.foldl( + function(acc, k) + local matchingPrefixes = [p for p in std.objectFields(routes) if std.startsWith(k, p)]; + if std.length(matchingPrefixes) > 0 then + local prefix = matchingPrefixes[0]; + acc + routes[prefix](prefix, k, pars[k]) + else + acc + { [k]:: pars[k] }, + std.objectFields(pars), + {} + ), + }, + + // Memory profiles + "memory-profile-low": import "profiles/memory-profile-low.jsonnet", + + // Model hosting + + "hosting-intel-battlemage-vllm": + import "model-hosting/intel-battlemage-vllm.jsonnet", + + "hosting-cpu-tgi": + import "model-hosting/cpu-tgi.jsonnet", + + "hosting-intel-xpu-tgi": + import "model-hosting/intel-xpu-tgi.jsonnet", + + "hosting-intel-gaudi-tgi": + import "model-hosting/intel-gaudi-tgi.jsonnet", + + "hosting-intel-xpu-vllm": + import "model-hosting/intel-xpu-vllm.jsonnet", + + "hosting-intel-gaudi-vllm": + import "model-hosting/intel-gaudi-vllm.jsonnet", + + "hosting-nvidia-gpu-vllm": + import "model-hosting/nvidia-gpu-vllm.jsonnet", + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/agent-manager-react.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/agent-manager-react.jsonnet new file mode 100644 index 00000000..954d80a7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/agent-manager-react.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "agent-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("agent-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "agent-manager-react", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "agent-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/chunker-recursive.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/chunker-recursive.jsonnet new file mode 100644 index 00000000..0b924aba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/chunker-recursive.jsonnet @@ -0,0 +1,55 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "chunk-size":: 2000, + "chunk-overlap":: 100, + + "chunker" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + std.toString($["chunk-size"]), + "--chunk-overlap", + std.toString($["chunk-overlap"]), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/configuration.jsonnet new file mode 100644 index 00000000..75afebb9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/configuration.jsonnet @@ -0,0 +1,57 @@ + +// This puts the default configuration together. References many things, +// flow classes, a default flow, token costs, prompts, agent tools + +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "init-trustgraph" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local cfgVol = engine.configVolume( + "trustgraph-cfg", "trustgraph", + { + "config.json": importstr "trustgraph/config.json", + } + ); + + local container = + engine.container("init-trustgraph") + .with_image(images.trustgraph_flow) + .with_command( + [ + "tg-init-trustgraph", + "-p", + url.pulsar_admin, + "--config-file", + "/trustgraph/config.json", + ] + ) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_volume_mount(cfgVol, "/trustgraph/"); + + local containerSet = engine.containers( + "init-trustgraph", [ container ] + ); + + engine.resources([ + cfgVol, + containerSet, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/document-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/document-rag.jsonnet new file mode 100644 index 00000000..e466e3a4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/document-rag.jsonnet @@ -0,0 +1,92 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "document-rag" +: { + + "doc-limit":: 20, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local docLimit = self["doc-limit"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "document-rag", + "-p", + url.pulsar, + "--doc-limit", + std.toString(docLimit), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "document-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "document-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/graph-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/graph-rag.jsonnet new file mode 100644 index 00000000..06ee5b3b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/graph-rag.jsonnet @@ -0,0 +1,283 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "kg-extract-definitions" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-definitions") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-definitions", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-definitions", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-relationships" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-relationships") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-relationships", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-relationships", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-agent" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-agent") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-agent", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-agent", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-ontology" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "300M", + "memory-reservation":: "300M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-ontology") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-ontology", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-ontology", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-rag" +: { + + concurrency:: 1, + "entity-limit":: 50, + "triple-limit":: 30, + "max-subgraph-size":: 400, + "max-path-length":: 2, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local entityLimit = self["entity-limit"]; + local tripleLimit = self["triple-limit"]; + local maxSubgraphSize = self["max-subgraph-size"]; + local maxPathLength = self["max-path-length"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("graph-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-rag", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--entity-limit", + std.toString(entityLimit), + "--triple-limit", + std.toString(tripleLimit), + "--max-subgraph-size", + std.toString(maxSubgraphSize), + "--max-path-length", + std.toString(maxPathLength), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "graph-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/librarian.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/librarian.jsonnet new file mode 100644 index 00000000..2ecda5a8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/librarian.jsonnet @@ -0,0 +1,58 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local garage = import "backends/garage.jsonnet"; +local cassandra = import "backends/cassandra.jsonnet"; + +{ + + "librarian" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("librarian") + .with_image(images.trustgraph_flow) + .with_command([ + "librarian", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + "--object-store-endpoint", + url.object_store, + "--object-store-access-key", + $.garage["access-key"], + "--object-store-secret-key", + $.garage["secret-key"], + "--object-store-region", + $.garage.region, + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "librarian", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +// Garage and Cassandra are used by the Librarian +} + garage + cassandra + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/mcp-server.jsonnet new file mode 100644 index 00000000..9bf7e3bd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/mcp-server.jsonnet @@ -0,0 +1,54 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "mcp-server" +: { + + port:: 8000, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local port = self.port; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("mcp-server-secret") + .with_env_var("MCP_SERVER_SECRET", "mcp-server-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local container = + engine.container("mcp-server") + .with_image(images.trustgraph_mcp) + .with_command([ + "mcp-server", + "--port", + std.toString(port), + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/prompt-overrides.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/prompt-overrides.jsonnet new file mode 100644 index 00000000..852ec09d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/prompt-overrides.jsonnet @@ -0,0 +1,24 @@ +local default_prompts = import "prompts/default-prompts.jsonnet"; + +{ + + with:: function(key, value) + if (key == "system-template") then + self + { + prompts +:: { + "system-template": value, + } + } + else + self + { + prompts +:: { + templates +:: { + [key] +:: { + prompt: value + } + } + } + }, + +} + default_prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/prompt-template.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/prompt-template.jsonnet new file mode 100644 index 00000000..3cd79d59 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/prompt-template.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "prompt" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("prompt") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "prompt", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "prompt-rag" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("prompt-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + "-p", + url.pulsar, + "--id", + "prompt-rag", + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "prompt-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/rev-gateway.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/rev-gateway.jsonnet new file mode 100644 index 00000000..fbd9f77a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/rev-gateway.jsonnet @@ -0,0 +1,62 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "rev-gateway" +: { + + // Invalid, but at least means the rev-gateway won't connect to anything + // it shouldn't. + token:: "INVALID_TOKEN", + uri:: "wss://127.0.0.1/api/v1/relay?token=" + self.token, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local uri = self.uri; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("rev-gateway-secret") + .with_env_var("REV_GATEWAY_SECRET", "rev-gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "rev-gateway", + "-p", + url.pulsar, + "--websocket-uri", + std.toString(uri), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/structured-data.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/structured-data.jsonnet new file mode 100644 index 00000000..52ea85da --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/structured-data.jsonnet @@ -0,0 +1,211 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "nlp-query" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("nlp-query") + .with_image(images.trustgraph_flow) + .with_command([ + "nlp-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "nlp-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-query" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("structured-query") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-query", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "structured-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-diag" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "96M", + "memory-reservation":: "96M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("structured-diag") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-diag", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "structured-diag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-rows" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-rows") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-rows", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-rows", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "row-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("row-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "row-embeddings", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "row-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/trustgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/trustgraph.jsonnet new file mode 100644 index 00000000..3cf12d26 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/core/trustgraph.jsonnet @@ -0,0 +1,473 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +local config_initialiser = import "configuration.jsonnet"; +local config = import "../runtime-config/trustgraph-config.jsonnet"; +local librarian = import "librarian.jsonnet"; +local mcp_server = import "mcp-server.jsonnet"; +local workbench = import "../ui/workbench-ui.jsonnet"; +local graphrag = import "graph-rag.jsonnet"; +local documentrag = import "document-rag.jsonnet"; +local prompt_template = import "prompt-template.jsonnet"; +local agent_manager = import "agent-manager-react.jsonnet"; +local structured_data = import "structured-data.jsonnet"; +local ddg = import "mcp/ddg-mcp-server.jsonnet"; + +// Helper to create a routing function for a target object +local route = function(target) + function(prefix, k, v) + local suffix = std.substr(k, std.length(prefix), std.length(k) - std.length(prefix)); + { [target] +: { [suffix]:: v } }; + +// Parameter prefix -> target object routing table +local routes = { + "prompt-rag-": route("prompt-rag"), + "prompt-": route("prompt"), + "text-completion-rag-": route("text-completion-rag"), + "text-completion-": route("text-completion"), + "embeddings-": route("embeddings"), + "api-gateway-": route("api-gateway"), + "chunk-": route("chunker"), + "graph-rag-": route("graph-rag"), + "graph-embeddings-": route("graph-embeddings"), + "kg-extract-definitions-": route("kg-extract-definitions"), + "kg-extract-relationships-": route("kg-extract-relationships"), + "kg-extract-agent-": route("kg-extract-agent"), + "kg-extract-ontology-": route("kg-extract-ontology"), + "kg-extract-objects-": route("kg-extract-objects"), + "garage-": route("garage"), + "config-svc-": route("config-svc"), + "pdf-decoder-": route("pdf-decoder"), + "mcp-tool-": route("mcp-tool"), + "mcp-server-": route("mcp-server"), + "metering-rag-": route("metering-rag"), + "metering-": route("metering"), + "kg-store-": route("kg-store"), + "kg-manager-": route("kg-manager"), + "librarian-": route("librarian"), + "agent-manager-": route("agent-manager"), + "document-rag-": route("document-rag"), + "document-embeddings-": route("document-embeddings"), + "rev-gateway-": route("rev-gateway"), + "nlp-query-": route("nlp-query"), + "structured-query-": route("structured-query"), + "structured-diag-": route("structured-diag"), + "init-trustgraph-": route("init-trustgraph"), +}; + +// Find longest matching prefix (most specific first) +local findRoute = function(k) + local prefixes = std.objectFields(routes); + local matching = std.filter(function(p) std.startsWith(k, p), prefixes); + local sorted = std.sort(matching, function(x) -std.length(x)); + if std.length(sorted) > 0 then sorted[0] else null; + +{ + + // Route parameters to appropriate internal objects based on prefix + with:: function(k, v) + local prefix = findRoute(k); + if prefix != null then + self + routes[prefix](prefix, k, v) + else + self + { [k]:: v }, + + "log-level":: "INFO", + + // Base objects with concurrency defaults (LLM/embeddings components merge into these) + "text-completion" +: { concurrency:: 1 }, + "text-completion-rag" +: { concurrency:: 1 }, + embeddings +: { concurrency:: 1 }, + + "api-gateway" +: { + + port:: 8088, + timeout:: 600, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local port = self.port; + local timeout = self.timeout; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("gateway-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "api-gateway", + "-p", + url.pulsar, + "--timeout", + std.toString(timeout), + "--port", + std.toString(port), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "chunker" +: { + + size:: 2000, + overlap:: 50, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local size = self.size; + local overlap = self.overlap; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + "-p", + url.pulsar, + "--chunk-size", + std.toString(size), + "--chunk-overlap", + std.toString(overlap), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "config-svc" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local cpuLimit = self["cpu-limit"]; + local cpuReservation = self["cpu-reservation"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("config-svc") + .with_image(images.trustgraph_flow) + .with_command([ + "config-svc", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(cpuLimit, memoryLimit) + .with_reservations(cpuReservation, memoryReservation); + + local containerSet = engine.containers( + "config-svc", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "pdf-decoder" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("pdf-decoder") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-decoder", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "pdf-decoder", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "mcp-tool" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("mcp-tool") + .with_image(images.trustgraph_flow) + .with_command([ + "mcp-tool", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "mcp-tool", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("metering") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "metering", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering-rag" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("metering-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + "-p", + url.pulsar, + "--id", + "metering-rag", + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "metering-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-store" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-store") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-store", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-store", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-manager", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + librarian + mcp_server + workbench + graphrag + + documentrag + prompt_template + agent_manager + structured_data + + config_initialiser + config + + ddg + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/embeddings/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/embeddings/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..37116de0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/embeddings/embeddings-fastembed.jsonnet @@ -0,0 +1,49 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-fastembed.jsonnet"; + +{ + + "fastembed-models":: models, + + "embeddings-models" +:: $["fastembed-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-fastembed", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/embeddings/embeddings-hf.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/embeddings/embeddings-hf.jsonnet new file mode 100644 index 00000000..492f05cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/embeddings/embeddings-hf.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-huggingface.jsonnet"; + +{ + + "huggingface-embeddings-models":: models, + + "embeddings-models" +:: $["huggingface-embeddings-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_hf) + .with_command([ + "embeddings-hf", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/embeddings/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/embeddings/embeddings-ollama.jsonnet new file mode 100644 index 00000000..05cbed72 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/embeddings/embeddings-ollama.jsonnet @@ -0,0 +1,52 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local models = import "parameters/embeddings-ollama.jsonnet"; + +{ + + "ollama-url":: "${OLLAMA_HOST}", + + "ollama-models":: models, + + "embeddings-models" +:: $["ollama-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-r", + $["ollama-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/aks-k8s.jsonnet new file mode 100644 index 00000000..07e050f2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/aks-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "disk.csi.azure.com", + parameters: { + // Standard disks (spinning magnetic), Locally Redundant Storage + // Cheapest, basically + skuName: "Standard_LRS", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/docker-compose.jsonnet new file mode 100644 index 00000000..334b3517 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/docker-compose.jsonnet @@ -0,0 +1,260 @@ +{ + + // Extract resources using the engine + package:: function(patterns) + std.foldl( + function(state, p) state + p.create(self), + std.objectValues(patterns), + {} + ), + + container:: function(name) + { + + local container = self, + + name:: name, + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_group:: function(x) self + + if std.objectHas(container, "group_add") then + { group_add: container.group_add + [x] } + else + { group_add: [x] }, + + with_command:: function(x) self + { + command: + if std.isString(x) then + std.strReplace(x, "$", "$$") + else if std.isArray(x) then + std.map(function(s) std.strReplace(s, "$", "$$"), x) + else + x + }, + + with_entrypoint:: function(x) self + { entrypoint: x }, + + with_runtime:: function(x) self + { runtime: x }, + + with_privileged:: function(x) self + { privileged: x }, + + with_ipc:: function(x) self + { ipc: x }, + + with_capability:: function(x) self + + if std.objectHas(container, "capability") then + { cap_add: container.capability + x } + else + { cap_add: [x], }, + + with_environment:: function(x) self + + if std.objectHas(container, "environment") then + { environment: container.environment + x } + else + { environment: x, }, + + with_device:: function(hdev, cdev) self + + if std.objectHas(container, "devices") then + { devices: container.devices + [ "%s:%s" % [hdev, cdev] ] } + else + { devices: [ "%s:%s" % [hdev, cdev] ], }, + + with_limits:: function(c, m) self + { + deploy +: { resources +: { + limits: { cpus: c, memory: m } + } }, + }, + + with_reservations:: function(c, m) self + { + deploy +: { resources +: { + reservations: { cpus: c, memory: m } + } }, + }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [vol.volid, mnt] + ] + else + [ + "%s:%s" % [vol.volid, mnt] + ] + }, + + with_bind_mount:: + function(src, dest) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [src, dest] + ] + else + [ + "%s:%s" % [src, dest] + ] + }, + + with_port:: + function(src, dest, name) + self + { + ports: + if std.objectHas(container, "ports") then + container.ports + [ "%d:%d" % [src, dest] ] + else + [ "%d:%d" % [src, dest] ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj.with_environment( + { [x]: "${" + x + "}" } + ), + vars.variables, + self + ), + + restart: "on-failure:100", + + add:: function() { + services +: { + [container.name]: container, + } + } + + }, + + internalService:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + with_size:: function(size) self + { size: size }, + + add:: function() { + volumes +: { + [volume.name]: {} + } + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: "./" + dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + variables:: [], + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + }, + + add:: function() { + } + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.foldl( + function(state, c) state + c.add(), + cont.containers, + {} + ), + + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/eks-k8s.jsonnet new file mode 100644 index 00000000..3fc3f035 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/eks-k8s.jsonnet @@ -0,0 +1,46 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "ebs.csi.aws.com", + parameters: { + type: "gp3", + encrypted: "true", + iops: "6000", + throughput: "400", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/gcp-k8s.jsonnet new file mode 100644 index 00000000..71792426 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/gcp-k8s.jsonnet @@ -0,0 +1,44 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "pd.csi.storage.gke.io", + parameters: { + type: "pd-balanced", + "csi.storage.k8s.io/fstype": "ext4", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/k8s.jsonnet new file mode 100644 index 00000000..2067f6ac --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/k8s.jsonnet @@ -0,0 +1,393 @@ +{ + + container:: function(name) + { + + local container = self, + + name: name, + limits: {}, + reservations: {}, + ports: [], + volumes: [], + bindMounts: [], + groups: [], + environment: [], + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_group:: function(x) self + { groups: super.groups + [x] }, + + with_privileged:: function(x) self + { privileged: x }, + + with_command:: function(x) self + { command: x }, + + with_entrypoint:: function(x) self + { entrypoint: x }, + + with_environment:: function(x) self + { + environment: super.environment + [ + { + name: v.key, value: v.value + } + for v in std.objectKeysValues(x) + ], + }, + + with_limits:: function(c, m) self + { limits: { cpu: c, memory: m } }, + + with_reservations:: + function(c, m) self + { reservations: { cpu: c, memory: m } }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: super.volumes + [{ + volume: vol, mount: mnt + }] + }, + + with_bind_mount:: + function(src, dest) + local name = "bind-" + std.strReplace(std.strReplace(src, "/", "-"), ".", "-"); + self + { + bindMounts: super.bindMounts + [{ + name: name, src: src, dest: dest + }] + }, + + with_port:: + function(src, dest, name) self + { + ports: super.ports + [ + { src: src, dest: dest, name : name } + ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj + { + environment: super.environment + [{ + name: x, + valueFrom: { + secretKeyRef: { + name: vars.name, + key: vars.keyMap[x], + } + } + }] + }, + vars.variables, + self + ), + + add:: function() [ + + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: container.name, + namespace: "trustgraph", + labels: { + app: container.name + } + }, + spec: { + replicas: 1, + selector: { + matchLabels: { + app: container.name, + } + }, + template: { + metadata: { + labels: { + app: container.name, + } + }, + spec: { + containers: [ + { + name: container.name, + image: container.image, + + // FIXME: Make everything run as + // root. Needed to get filesystems + // to be accessible. There's a + // better way of doing this? + securityContext: { + runAsUser: 0, + runAsGroup: 0, + } + ( + if std.objectHas(container, "privileged") && container.privileged then + { privileged: true } + else {} + ), + + resources: { + requests: container.reservations, + limits: container.limits + }, + } + ( + if std.length(container.ports) > 0 then + { + ports: [ + { + hostPort: port.src, + containerPort: port.dest, + } + for port in container.ports + ] + } else + {}) + + + (if std.objectHas(container, "entrypoint") then + // Entrypoint is set - use command for entrypoint, args for command + (if std.isString(container.entrypoint) && container.entrypoint == "" then + { command: [] } + else if std.isArray(container.entrypoint) then + { command: container.entrypoint } + else + { command: [container.entrypoint] } + ) + (if std.objectHas(container, "command") then + { args: container.command } + else {}) + else if std.objectHas(container, "command") then + { command: container.command } + else {}) + + + (if std.length(container.environment) > 0 then + { + env: container.environment, + } + else {}) + + + (if std.length(container.volumes) > 0 || std.length(container.bindMounts) > 0 then + { + volumeMounts: [ + { + mountPath: vol.mount, + name: vol.volume.name, + } + for vol in container.volumes + ] + [ + { + mountPath: bm.dest, + name: bm.name, + } + for bm in container.bindMounts + ] + } + + else + {} + ) + ], + volumes: [ + vol.volume.volRef() + for vol in container.volumes + ] + [ + { + name: bm.name, + hostPath: { path: bm.src } + } + for bm in container.bindMounts + ] + } + ( + if std.length(container.groups) > 0 then + { securityContext: { supplementalGroups: container.groups } } + else {} + ) + }, + } + {} + + } + + ] + + }, + + // Just an alias + internalService:: self.service, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + ports: [], + + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + + add:: function() [ + + { + + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + storageClassName: "tg", + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + data: parts + }, + ], + + + volRef:: function() { + name: volume.name, + configMap: { name: volume.name }, + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + variables: [], + keyMap: {}, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + }, + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + keyMap: super.keyMap + { [name]: key }, + }, + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.flattenArrays( + [ c.add() for c in cont.containers ] + ), + + }, + + resources:: function(res) + + std.flattenArrays( + [ c.add() for c in res ] + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/minikube-k8s.jsonnet new file mode 100644 index 00000000..858b17ad --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/minikube-k8s.jsonnet @@ -0,0 +1,115 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList, + + volume:: function(name) + { + local volume = self, + name: name, + with_size:: function(size) self + { size: size }, + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolume", + metadata: { + name: volume.name, + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + capacity: { + storage: volume.size, + }, + persistentVolumeReclaimPolicy: "Delete", + hostPath: { + path: "/data/pv-" + volume.name, + }, + } + }, + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + service:: function(containers) + { + local service = self, + name: containers.name, + ports: [], + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + add:: function() [ + { + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + type: "LoadBalancer", + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/noop.jsonnet new file mode 100644 index 00000000..1f384648 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/noop.jsonnet @@ -0,0 +1,79 @@ +{ + + // Extract resources usnig the engine + package:: function(patterns) {}, + + container:: function(name) { + + with_image:: function(x) self + {}, + + with_user:: function(x) self + {}, + + with_command:: function(x) self + {}, + + with_runtime:: function(x) self + {}, + + with_privileged:: function(x) self + {}, + + with_ipc:: function(x) self + {}, + + with_capability:: function(x) self + {}, + + with_environment:: function(x) self + {}, + + with_device:: function(hdev, cdev) self + {}, + + with_limits:: function(c, m) self + {}, + + with_reservations:: function(c, m) self + {}, + + with_volume_mount:: self + {}, + + with_port:: function(src, dest, name) self + {}, + + with_env_var_secrets:: function(vars) self + {}, + + add:: function() {}, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + service:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + volume:: function(name) { + with_size:: function(size) self + {}, + add:: function() {}, + }, + + configVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + secretVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self + {}, + add:: function() {}, + }, + + containers:: function(name, containers) { + add:: function() {}, + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/ovh-k8s.jsonnet new file mode 100644 index 00000000..15d57c83 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/ovh-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "cinder.csi.openstack.org", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", + parameters: { + availability: "nova", + fsType: "ext4", + type: "high-speed", + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/scw-k8s.jsonnet new file mode 100644 index 00000000..eed23968 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/engine/scw-k8s.jsonnet @@ -0,0 +1,40 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "csi.scaleway.com", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/agent-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/agent-extract.jsonnet new file mode 100644 index 00000000..4c5785c8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/agent-extract.jsonnet @@ -0,0 +1,35 @@ +// Agent-based extraction module +// Uses AI agents for more sophisticated knowledge extraction from text +// Leverages agent tools and reasoning for complex extraction tasks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - internal agent extraction service + "interfaces" +: { + }, + + // No configurable parameters for agent extraction + "parameters" +: { + }, + + // Flow-level processors for agent-based extraction + "flow" +: { + // Agent-based knowledge extraction processor + // Uses AI agents with tools to extract structured knowledge + "kg-extract-agent:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output knowledge triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "agent-request": request("agent:{id}"), // Agent service requests + "agent-response": response("agent:{id}"), // Agent service responses + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/agent.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/agent.jsonnet new file mode 100644 index 00000000..2baa31d2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/agent.jsonnet @@ -0,0 +1,60 @@ +// Agent management module +// Provides AI agent orchestration and tool integration +// Manages agent conversations, tool calls, and response coordination +// Supports MCP tools, GraphRAG, and structured queries + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services (agent requires LLM for reasoning, MCP for tools) +local llm_services = import "llm-services.jsonnet"; +local mcp_service = import "mcp-service.jsonnet"; + +// Merge shared services with agent-specific configuration +llm_services + mcp_service + { + + // External interfaces for agent operations + "interfaces" +: { + "agent": request_response("agent:{id}"), + }, + + // Flow-level processors for agent management + "flow" +: { + // Agent manager orchestrates agent conversations and tool usage + "agent-manager:{id}": { + + // Agent communication channels + request: request("agent:{id}"), + next: request("agent:{id}"), + response: response("agent:{id}"), + + // LLM and prompt services + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + + // Tool integrations + "mcp-tool-request": request("mcp-tool:{id}"), + "mcp-tool-response": response("mcp-tool:{id}"), + "graph-rag-request": request("graph-rag:{id}"), + "graph-rag-response": response("graph-rag:{id}"), + "structured-query-request": request("structured-query:{id}"), + "structured-query-response": response("structured-query:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "row-embeddings-query-request": request("row-embeddings:{id}"), + "row-embeddings-query-response": response("row-embeddings:{id}"), + + // Explainability + explainability: flow("triples-store:{id}"), + }, + }, + + // Blueprint-level processors for agent-related services + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/document-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/document-store.jsonnet new file mode 100644 index 00000000..769718fa --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/document-store.jsonnet @@ -0,0 +1,57 @@ +// Document store module +// Infrastructure for document-based RAG using chunk embeddings +// Handles document embedding storage, retrieval, and question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with document store configuration +llm_services + embeddings_service + { + + // External interfaces for document store + "interfaces" +: { + // Document embedding storage and retrieval + "document-embeddings-store": flow("document-embeddings-store:{id}"), + "document-rag": request_response("document-rag:{id}"), + "document-embeddings": request_response("document-embeddings:{id}"), + }, + + // Flow-level processors for document embedding and storage + "flow" +: { + "document-embeddings:{id}": { + input: flow("chunk-load:{id}"), + output: flow("document-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "doc-embeddings-write:{id}": { + input: flow("document-embeddings-store:{id}"), + }, + "document-rag:{id}": { + request: request("document-rag:{id}"), + response: response("document-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "document-embeddings-request": request("document-embeddings:{id}"), + "document-embeddings-response": response("document-embeddings:{id}"), + explainability: flow("triples-store:{id}"), + }, + "doc-embeddings-query:{id}": { + request: request("document-embeddings:{id}"), + response: response("document-embeddings:{id}"), + }, + }, + + // Blueprint-level processors for document RAG operations + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/embeddings-service.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/embeddings-service.jsonnet new file mode 100644 index 00000000..1537e6c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/embeddings-service.jsonnet @@ -0,0 +1,30 @@ +// Shared embeddings service module +// Provides vector embedding generation for text +// Import this module in any flow that requires embeddings + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // Interfaces exposed by embeddings service + "interfaces" +: { + "embeddings": request_response("embeddings:{id}"), + }, + + "parameters" +: { + }, + + // Flow-level processor for embeddings + "flow" +: { + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/flow-blueprints.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/flow-blueprints.jsonnet new file mode 100644 index 00000000..8f054690 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/flow-blueprints.jsonnet @@ -0,0 +1,48 @@ +// TrustGraph Flow Blueprints Configuration +// +// RAG Modes (4 types): +// - Document RAG: Uses document chunk embeddings +// - Graph RAG: Extracts definitions + relationships to graph +// - Ontology RAG: Extracts using ontology definitions to graph (mutually exclusive with Graph RAG) +// - Structured RAG: Extracts objects to object store +// +// Module structure: +// - *-store: Storage and query infrastructure +// - *-extract: Extraction methods + +// Import all the modular flow components +local graph_store = import "graph-store.jsonnet"; +local document_store = import "document-store.jsonnet"; +local structured_store = import "structured-store.jsonnet"; +local graphrag_extract = import "graphrag-extract.jsonnet"; +local ontorag_extract = import "ontorag-extract.jsonnet"; +local structured_extract = import "structured-extract.jsonnet"; +local agent = import "agent.jsonnet"; +local load = import "load.jsonnet"; +local kgcore = import "kgcore.jsonnet"; + +{ + + // Full system: Graph RAG + Document RAG + knowledge cores + "everything": { + description: "Graph RAG + Document RAG + knowledge cores", + tags: ["document-rag", "graph-rag", "kgcore"], + } + + graph_store + document_store + agent + load + + graphrag_extract + kgcore + structured_store, + + // Structured RAG only + "structured": { + description: "Structured data extraction and querying", + tags: ["structured"], + } + + structured_store + structured_extract + agent + load, + + // Ontology RAG + knowledge cores + "ontology": { + description: "Ontology RAG + knowledge cores", + tags: ["onto-rag", "kgcore"], + } + + graph_store + ontorag_extract + agent + load + kgcore, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/graph-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/graph-store.jsonnet new file mode 100644 index 00000000..64546a51 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/graph-store.jsonnet @@ -0,0 +1,73 @@ +// Graph store module +// Shared infrastructure for graph-based RAG (used by both GraphRAG and OntologyRAG) +// Handles knowledge graph storage, embeddings, and graph-based question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with graph store configuration +llm_services + embeddings_service + { + + // External interfaces exposed by the graph store + "interfaces" +: { + // Data ingestion interfaces for graph construction + "entity-contexts-load": flow("entity-contexts-load:{id}"), + "triples-store": flow("triples-store:{id}"), + "graph-embeddings-store": flow("graph-embeddings-store:{id}"), + + // Query interfaces for graph-based operations + "graph-rag": request_response("graph-rag:{id}"), + "triples": request_response("triples:{id}"), + "graph-embeddings": request_response("graph-embeddings:{id}"), + }, + + // Flow-level processors - handle data streams for a specific flow instance + "flow" +: { + "graph-embeddings:{id}": { + input: flow("entity-contexts-load:{id}"), + output: flow("graph-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "triples-write:{id}": { + input: flow("triples-store:{id}"), + }, + "graph-embeddings-write:{id}": { + input: flow("graph-embeddings-store:{id}"), + }, + "graph-rag:{id}": { + request: request("graph-rag:{id}"), + response: response("graph-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "graph-embeddings-request": request("graph-embeddings:{id}"), + "graph-embeddings-response": response("graph-embeddings:{id}"), + "triples-request": request("triples:{id}"), + "triples-response": response("triples:{id}"), + explainability: flow("triples-store:{id}"), + "librarian-request": request("librarian"), + "librarian-response": response("librarian"), + }, + "triples-query:{id}": { + request: request("triples:{id}"), + response: response("triples:{id}"), + }, + "graph-embeddings-query:{id}": { + request: request("graph-embeddings:{id}"), + response: response("graph-embeddings:{id}"), + }, + }, + + // Blueprint-level processors - shared across all flow instances of this blueprint + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/graphrag-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/graphrag-extract.jsonnet new file mode 100644 index 00000000..6bcbca14 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/graphrag-extract.jsonnet @@ -0,0 +1,44 @@ +// GraphRAG extraction module +// Extraction method for GraphRAG - extracts definitions and relationships +// Mutually exclusive with OntologyRAG extraction (both write to graph store) + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts entity definitions from text chunks + // Identifies and defines key entities mentioned in the text + "kg-extract-definitions:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output definition triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition extraction prompts + "prompt-response": response("prompt:{id}"), + }, + + // Extracts relationships between entities + // Identifies how entities are connected and interact + "kg-extract-relationships:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output relationship triples + "prompt-request": request("prompt:{id}"), // Relationship extraction prompts + "prompt-response": response("prompt:{id}"), + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/helpers.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/helpers.jsonnet new file mode 100644 index 00000000..eabb3bf4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/helpers.jsonnet @@ -0,0 +1,29 @@ +// Helper functions for flow configuration +// Provides utility functions for constructing flow, request, and response URIs +// used throughout the TrustGraph flow configuration system + +// Creates a persistent flow URI for data streams +// Persistent flows retain messages until consumed +local flow(x) = "persistent://tg/flow/" + x; + +// Creates a non-persistent request URI for request-response patterns +// Non-persistent means messages are not retained if no consumer is present +local request(x) = "non-persistent://tg/request/" + x; + +// Creates a non-persistent response URI for request-response patterns +local response(x) = "non-persistent://tg/response/" + x; + +// Creates a request-response pair for bidirectional communication +// Returns an object with both request and response URIs +local request_response(x) = { + request: request(x), + response: response(x), +}; + +// Export all helper functions for use in other modules +{ + flow: flow, + request: request, + response: response, + request_response: request_response, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/kgcore.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/kgcore.jsonnet new file mode 100644 index 00000000..bfc01ada --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/kgcore.jsonnet @@ -0,0 +1,31 @@ +// Knowledge Graph Core storage module +// Handles persistent storage of knowledge graph data +// Consolidates triples and graph embeddings into permanent storage +// Creates the core knowledge base for long-term use + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; + +{ + // No external interfaces - internal storage service + "interfaces" +: { + }, + + // No configurable parameters for core storage + "parameters" +: { + }, + + // Flow-level processors for knowledge graph storage + "flow" +: { + // Knowledge graph store consolidates extracted knowledge + // Takes processed triples and embeddings and stores them permanently + "kg-store:{id}": { + "triples-input": flow("triples-store:{id}"), // Input RDF triples stream + "graph-embeddings-input": flow("graph-embeddings-store:{id}"), // Input graph embeddings + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/llm-parameters.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/llm-parameters.jsonnet new file mode 100644 index 00000000..a99f72ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/llm-parameters.jsonnet @@ -0,0 +1,59 @@ +{ + + // LLM model selection for normal LLM + "llm-model": { + "type": "llm-model", + "description": "LLM model", + "order": 1, + "advanced": false, + }, + + // LLM model for RAG operations + "llm-rag-model": { + "type": "llm-model", + "description": "LLM model for RAG", + "order": 2, + "advanced": true, + "controlled-by": "llm-model", + }, + + // LLM model selection for normal LLM + "llm-temperature": { + "type": "llm-temperature", + "description": "LLM temperature", + "order": 3, + "advanced": true, + }, + + // LLM model selection for normal LLM + "llm-rag-temperature": { + "type": "llm-temperature", + "description": "LLM temperature for RAG", + "order": 4, + "advanced": true, + }, + + "embeddings-model": { + "type": "embeddings-model", + "description": "Embeddings model", + "order": 5, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-size": { + "type": "chunk-size", + "description": "Chunk size", + "order": 6, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-overlap": { + "type": "chunk-overlap", + "description": "Chunk overlap", + "order": 7, + "advanced": true, + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/llm-services.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/llm-services.jsonnet new file mode 100644 index 00000000..ee6e09ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/llm-services.jsonnet @@ -0,0 +1,66 @@ +// Shared LLM services module +// Provides text completion, prompt processing, and metering services +// Import this module in any flow that requires LLM functionality + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // Interfaces exposed by LLM services + "interfaces" +: { + "prompt": request_response("prompt:{id}"), + "text-completion": request_response("text-completion:{id}"), + }, + + // LLM configuration parameters + "parameters" +: llm_parameters, + + // Flow-level processors for LLM services + "flow" +: { + // Primary text completion service + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + + // RAG-specific text completion (may use different model) + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + + // Prompt processing service + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + + // RAG-specific prompt processing + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + + // Usage metering for primary completion + "metering:{id}": { + input: response("text-completion:{id}"), + }, + + // Usage metering for RAG completion + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/load.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/load.jsonnet new file mode 100644 index 00000000..ea7c74d4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/load.jsonnet @@ -0,0 +1,51 @@ +// Document loading and preprocessing module +// Handles document ingestion, format conversion, and chunking +// Converts PDFs to text and splits documents into processable chunks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services (load requires embeddings for chunk processing) +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with load-specific configuration +embeddings_service + { + + // External interfaces for document loading + "interfaces" +: { + "document-load": flow("document-load:{id}"), + "text-load": flow("text-document-load:{id}"), + }, + + // Flow-level processors for document preprocessing + "flow" +: { + // PDF decoder converts PDF documents to text + // Also emits page provenance triples and saves pages via librarian + "pdf-decoder:{id}": { + input: flow("document-load:{id}"), + output: flow("text-document-load:{id}"), + triples: flow("triples-store:{id}"), + "librarian-request": request("librarian"), + "librarian-response": response("librarian"), + }, + + // Chunker splits documents into smaller, processable pieces + // Also emits chunk provenance triples and saves chunks via librarian + "chunker:{id}": { + input: flow("text-document-load:{id}"), + output: flow("chunk-load:{id}"), + triples: flow("triples-store:{id}"), + "librarian-request": request("librarian"), + "librarian-response": response("librarian"), + "chunk-size": "{chunk-size}", + "chunk-overlap": "{chunk-overlap}", + }, + }, + + // Blueprint-level processors for document loading services + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/mcp-service.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/mcp-service.jsonnet new file mode 100644 index 00000000..5d599a67 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/mcp-service.jsonnet @@ -0,0 +1,31 @@ +// Shared MCP (Model Context Protocol) tool service module +// Provides MCP tool execution capabilities for agents +// Import this module in any flow that requires MCP tool integration + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // Interfaces exposed by MCP service + "interfaces" +: { + "mcp-tool": request_response("mcp-tool:{id}"), + }, + + "parameters" +: { + }, + + // Flow-level processor for MCP tool execution + "flow" +: { + "mcp-tool:{id}": { + request: request("mcp-tool:{id}"), + response: response("mcp-tool:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/ontorag-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/ontorag-extract.jsonnet new file mode 100644 index 00000000..e5521d8a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/ontorag-extract.jsonnet @@ -0,0 +1,39 @@ +// OntologyRAG extraction module +// Extraction method for OntologyRAG - extracts using ontology definitions +// Mutually exclusive with GraphRAG extraction (both write to graph store) + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts using ontology definitions + "kg-extract-ontology:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition + // extraction prompts + "prompt-response": response("prompt:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/structured-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/structured-extract.jsonnet new file mode 100644 index 00000000..54f6a05d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/structured-extract.jsonnet @@ -0,0 +1,30 @@ +// Structured RAG extraction module +// Extracts structured rows from text chunks +// Outputs to rows-store for structured data querying + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + "interfaces" +: { + }, + + "parameters" +: { + }, + + // Flow-level processor for structured row extraction + "flow" +: { + "kg-extract-rows:{id}": { + input: flow("chunk-load:{id}"), + output: flow("rows-store:{id}"), + "entity-contexts": flow("entity-contexts-load:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/structured-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/structured-store.jsonnet new file mode 100644 index 00000000..3668222d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/flows/structured-store.jsonnet @@ -0,0 +1,78 @@ +// Structured store module +// Shared infrastructure for structured data RAG +// Handles row storage, retrieval, and NLP query capabilities + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with structured store configuration +llm_services + embeddings_service + { + + // External interfaces for structured store + "interfaces" +: { + // Row storage and querying + "rows-store": flow("rows-store:{id}"), + "row-embeddings-store": flow("row-embeddings-store:{id}"), + "rows": request_response("rows:{id}"), + "row-embeddings": request_response("row-embeddings:{id}"), + + // Query interfaces + "nlp-query": request_response("nlp-query:{id}"), + "structured-query": request_response("structured-query:{id}"), + "structured-diag": request_response("structured-diag:{id}"), + }, + + // Flow-level processors for structured storage and query + "flow" +: { + "row-embeddings:{id}": { + input: flow("rows-store:{id}"), + output: flow("row-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "rows-write:{id}": { + input: flow("rows-store:{id}"), + }, + "row-embeddings-write:{id}": { + input: flow("row-embeddings-store:{id}"), + }, + "rows-query:{id}": { + request: request("rows:{id}"), + response: response("rows:{id}"), + }, + "row-embeddings-query:{id}": { + request: request("row-embeddings:{id}"), + response: response("row-embeddings:{id}"), + }, + "nlp-query:{id}": { + request: request("nlp-query:{id}"), + response: response("nlp-query:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + }, + "structured-query:{id}": { + request: request("structured-query:{id}"), + response: response("structured-query:{id}"), + "nlp-query-request": request("nlp-query:{id}"), + "nlp-query-response": response("nlp-query:{id}"), + "rows-query-request": request("rows:{id}"), + "rows-query-response": response("rows:{id}"), + }, + "structured-diag:{id}": { + request: request("structured-diag:{id}"), + response: response("structured-diag:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/azure-openai.jsonnet new file mode 100644 index 00000000..bee0fe47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/azure-openai.jsonnet @@ -0,0 +1,112 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure-openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-openai-" + key]:: value, + }, + +// Strategy is to specify the model with the AZURE_MODEL environment +// variable. This isn't something that can just be specified dynamically, +// it has to match what was provisioned in Azure. + + "azure-openai-max-output-tokens":: 4192, + "azure-openai-temperature":: 0.0, + "azure-openai-models":: models, + + "llm-models" +:: $["azure-openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/azure.jsonnet new file mode 100644 index 00000000..f4db7500 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/azure.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-" + key]:: value, + }, + + "azure-max-output-tokens":: 4096, + "azure-temperature":: 0.0, + "azure-models":: models, + + "llm-models" +:: $["azure-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/bedrock.jsonnet new file mode 100644 index 00000000..0db1475b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/bedrock.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/bedrock.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["bedrock-" + key]:: value, + }, + + "bedrock-max-output-tokens":: 4096, + "bedrock-temperature":: 0.0, + "bedrock-models":: models, + + "llm-models" +:: $["bedrock-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/claude.jsonnet new file mode 100644 index 00000000..f3125e96 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/claude.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/claude.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["claude-" + key]:: value, + }, + + "claude-max-output-tokens":: 4096, + "claude-temperature":: 0.0, + "claude-models":: models, + + "llm-models" +:: $["claude-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/cohere.jsonnet new file mode 100644 index 00000000..6517f7a0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/cohere.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/cohere.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["cohere-" + key]:: value, + }, + + "cohere-temperature":: 0.0, + "cohere-models":: models, + + "llm-models" +:: $["cohere-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/googleaistudio.jsonnet new file mode 100644 index 00000000..05a8592c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/googleaistudio.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/googleaistudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["googleaistudio-" + key]:: value, + }, + + "googleaistudio-max-output-tokens":: 4096, + "googleaistudio-temperature":: 0.0, + "googleaistudio-models":: models, + + "llm-models" +:: $["googleaistudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-googleaistudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/llamafile.jsonnet new file mode 100644 index 00000000..f3bee2cc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/llamafile.jsonnet @@ -0,0 +1,94 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/slm.jsonnet"; +local models = import "parameters/llamafile.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["llamafile-" + key]:: value, + }, + + "llamafile-models":: models, + + "llm-models" +:: $["llamafile-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/lmstudio.jsonnet new file mode 100644 index 00000000..4cf78caa --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/lmstudio.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/lmstudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["lmstudio-" + key]:: value, + }, + + "lmstudio-max-output-tokens":: 4096, + "lmstudio-temperature":: 0.0, + "lmstudio-models":: models, + + "llm-models" +:: $["lmstudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/mistral.jsonnet new file mode 100644 index 00000000..2fb8c562 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/mistral.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/mistral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "mistral-max-output-tokens":: 4096, + "mistral-temperature":: 0.0, + "mistral-models":: models, + + "llm-models" +:: $["mistral-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/ollama.jsonnet new file mode 100644 index 00000000..6143f5cc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/ollama.jsonnet @@ -0,0 +1,102 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/ollama.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["ollama-" + key]:: value, + }, + + "ollama-models":: models, + + "llm-models" +:: $["ollama-models"], + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/openai.jsonnet new file mode 100644 index 00000000..c3d482fd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/openai.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["openai-" + key]:: value, + }, + + "openai-max-output-tokens":: 4096, + "openai-temperature":: 0.0, + "openai-models":: models, + + "llm-models" +:: $["openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/tgi.jsonnet new file mode 100644 index 00000000..91588dff --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/tgi.jsonnet @@ -0,0 +1,108 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-" + key]:: value, + }, + + "tgi-max-output-tokens":: 1024, + "tgi-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/vertexai.jsonnet new file mode 100644 index 00000000..cb8c141f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/vertexai.jsonnet @@ -0,0 +1,120 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vertexai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vertexai-" + key]:: value, + }, + + "vertexai-private-key":: "/vertexai/private.json", + "vertexai-region":: "us-central1", + "vertexai-max-output-tokens":: 4096, + "vertexai-temperature":: 0.0, + "vertexai-models":: models, + + "llm-models" +:: $["vertexai-models"], + + "text-completion" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion-rag") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/vllm.jsonnet new file mode 100644 index 00000000..8b846bbf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/llm/vllm.jsonnet @@ -0,0 +1,113 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vllm.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-" + key]:: value, + }, + + "vllm-models":: models, + + "llm-models" +:: $["vllm-models"], + + "vllm-max-output-tokens":: 1024, + "vllm-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + "-p", + url.pulsar, + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/mcp/ddg-mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/mcp/ddg-mcp-server.jsonnet new file mode 100644 index 00000000..04176f15 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/mcp/ddg-mcp-server.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "ddg-mcp-server-port":: 9870, + + "ddg-mcp-server" +: { + + create:: function(engine) + + local port = $["ddg-mcp-server-port"]; + + local container = + engine.container("ddg-mcp-server") + .with_image(images["ddg-mcp-server"]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "ddg-mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + mcp +:: { + "duckduckgo": { + "remote-name": "search", + local port = $["ddg-mcp-server-port"], + local url = "http://ddg-mcp-server:%s/mcp" % port, + "url": url, + } + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/cpu-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/cpu-tgi.jsonnet new file mode 100644 index 00000000..04b61566 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/cpu-tgi.jsonnet @@ -0,0 +1,68 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + "tgi-service-storage":: "20G", + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-cpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--cuda-graphs", + "0", + ]) + .with_environment({ + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-battlemage-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-battlemage-vllm.jsonnet new file mode 100644 index 00000000..7646398c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-battlemage-vllm.jsonnet @@ -0,0 +1,98 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-Nemo-Instruct-2407", + "vllm-service-cpus":: "32.0", + "vllm-service-memory":: "48G", + "vllm-service-storage":: "48G", + "vllm-service-tokenizer-mode":: "mistral", + "vllm-service-datatype":: "float16", + "vllm-service-quantization":: "woq_int4", + "vllm-service-hf-token":: null, + "vllm-service-max-model-len":: 8192, + "vllm-service-max-num-seqs":: 16, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-battlemage"]) + .with_entrypoint("") // Clear default entrypoint + .with_command([ + "python", + "-m", + "ipex_llm.vllm.xpu.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--device", + "xpu", + "--dtype", + $["vllm-service-datatype"], + "--enforce-eager", + "--max-model-len", + std.toString($["vllm-service-max-model-len"]), + "--max-num-seqs", + std.toString($["vllm-service-max-num-seqs"]), + "--load-in-low-bit", + $["vllm-service-quantization"], + "--trust-remote-code", + "--tokenizer-mode", + $["vllm-service-tokenizer-mode"], + "--disable-sliding-window", + ]) + .with_environment({ + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS: "1", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-gaudi-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-gaudi-tgi.jsonnet new file mode 100644 index 00000000..5af36b78 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-gaudi-tgi.jsonnet @@ -0,0 +1,96 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "meta-llama/Llama-3.3-70B-Instruct", + "tgi-service-cpus":: "64.0", + "tgi-service-memory":: "64G", + "tgi-service-storage":: "50G", + "tgi-service-num-shard":: 8, + "tgi-service-max-input-tokens":: 4096, + "tgi-service-max-total-tokens":: 4096, + "tgi-service-max-batch-size":: 128, + "tgi-service-max-concurrent-requests":: 512, + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-gaudi"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--sharded", + "true", + "--num-shard", + std.toString($["tgi-service-num-shard"]), + "--max-input-tokens", + std.toString($["tgi-service-max-input-tokens"]), + "--max-total-tokens", + std.toString($["tgi-service-max-total-tokens"]), + "--max-batch-size", + std.toString($["tgi-service-max-batch-size"]), + "--max-waiting-tokens", + "7", + "--max-concurrent-requests", + std.toString($["tgi-service-max-concurrent-requests"]), + "--cuda-graphs", + "0", + ]) + .with_runtime("habana") + .with_environment({ + HABANA_VISIBLE_DEVICES: "all", + OMPI_MCA_btl_vader_single_copy_mechanism: "none", + ENABLE_HPU_GRAPH: "true", + LIMIT_HPU_GRAPH: "true", + USE_FLASH_ATTENTION: "true", + FLASH_ATTENTION_RECOMPUTE: "true", + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-gaudi-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-gaudi-vllm.jsonnet new file mode 100644 index 00000000..81d31c14 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-gaudi-vllm.jsonnet @@ -0,0 +1,76 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "64.0", + "vllm-service-memory":: "64G", + "vllm-service-storage":: "50G", + "vllm-service-tensor-parallel-size":: 8, + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-gaudi"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--tensor-parallel-size", + std.toString($["vllm-service-tensor-parallel-size"]), + ]) + .with_runtime("habana") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + HABANA_VISIBLE_DEVICES: "all", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-xpu-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-xpu-tgi.jsonnet new file mode 100644 index 00000000..bc806874 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-xpu-tgi.jsonnet @@ -0,0 +1,75 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + "tgi-service-storage":: "20G", + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-intel-xpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--cuda-graphs", + "0", + ]) + .with_environment({ + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_group("video") + .with_group("render") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-xpu-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-xpu-vllm.jsonnet new file mode 100644 index 00000000..f6e82905 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/intel-xpu-vllm.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "8.0", + "vllm-service-memory":: "16G", + "vllm-service-storage":: "20G", + "vllm-service-datatype":: "float16", + "vllm-service-max-model-len":: 4096, + "vllm-service-max-num-seqs":: 16, + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-xpu"]) + .with_command([ + "python", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--device", + "xpu", + "--dtype", + $["vllm-service-datatype"], + "--enforce-eager", + "--max-model-len", + std.toString($["vllm-service-max-model-len"]), + "--max-num-seqs", + std.toString($["vllm-service-max-num-seqs"]), + "--block-size", + "64", + "--gpu-memory-util", + "0.85", + "--trust-remote-code", + "--disable-sliding-window", + ]) + .with_environment({ + VLLM_USE_V1: "1", + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_group("video") + .with_group("render") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/nvidia-gpu-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/nvidia-gpu-vllm.jsonnet new file mode 100644 index 00000000..d04876c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/model-hosting/nvidia-gpu-vllm.jsonnet @@ -0,0 +1,72 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-7B-Instruct-v0.3", + "vllm-service-cpus":: "0.5", + "vllm-service-memory":: "1G", + "vllm-service-storage":: "50G", + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-nvidia"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + ]) + .with_runtime("nvidia") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/monitoring/grafana.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/monitoring/grafana.jsonnet new file mode 100644 index 00000000..dd0dddd7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/monitoring/grafana.jsonnet @@ -0,0 +1,124 @@ +local images = import "values/images.jsonnet"; +local loki = import "loki.jsonnet"; + +{ + + "prometheus" +: { + + create:: function(engine) + + local vol = engine.volume("prometheus-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "prometheus-cfg", "prometheus", + { + "prometheus.yml": importstr "prometheus/prometheus.yml", + } + ); + + local container = + engine.container("prometheus") + .with_image(images.prometheus) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M") + .with_port(9090, 9090, "http") + .with_volume_mount(cfgVol, "/etc/prometheus/") + .with_volume_mount(vol, "/prometheus"); + + local containerSet = engine.containers( + "prometheus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9090, 9090, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + + "grafana" +: { + + create:: function(engine) + + local vol = engine.volume("grafana-storage").with_size("20G"); + + local provDashVol = engine.configVolume( + "prov-dash", "grafana/provisioning/", + { + "dashboard.yml": + importstr "grafana/provisioning/dashboard.yml", + } + + ); + + local provDataVol = engine.configVolume( + "prov-data", "grafana/provisioning/", + { + "datasource.yml": + importstr "grafana/provisioning/datasource.yml", + } + + ); + + local dashVol = engine.configVolume( + "dashboards", "grafana/dashboards/", + { + "overview-dashboard.json": + importstr "grafana/dashboards/overview-dashboard.json", + "log-dashboard.json": + importstr "grafana/dashboards/log-dashboard.json", + } + + ); + + local container = + engine.container("grafana") + .with_image(images.grafana) + .with_environment({ + // GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin", + // GF_AUTH_ANONYMOUS_ENABLED: "true", + // GF_ORG_ROLE: "Admin", + GF_ORG_NAME: "trustgraph.ai", + // GF_SERVER_ROOT_URL: "https://example.com", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(3000, 3000, "cassandra") + .with_volume_mount(vol, "/var/lib/grafana") + .with_volume_mount( + provDashVol, "/etc/grafana/provisioning/dashboards/" + ) + .with_volume_mount( + provDataVol, "/etc/grafana/provisioning/datasources/" + ) + .with_volume_mount( + dashVol, "/var/lib/grafana/dashboards/" + ); + + local containerSet = engine.containers( + "grafana", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3000, 3000, "http"); + + engine.resources([ + vol, + provDashVol, + provDataVol, + dashVol, + containerSet, + service, + ]) + + }, + +} + loki + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/monitoring/loki.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/monitoring/loki.jsonnet new file mode 100644 index 00000000..0774f6cf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/monitoring/loki.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "loki" +: { + + create:: function(engine) + + local vol = engine.volume("loki-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "loki-cfg", "loki", + { + "local-config.yaml": importstr "loki/local-config.yaml", + } + ); + + local container = + engine.container("loki") + .with_image(images.loki) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(3100, 3100, "http") + .with_volume_mount(cfgVol, "/etc/loki/") + .with_volume_mount(vol, "/loki"); + + local containerSet = engine.containers( + "loki", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3100, 3100, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/ocr/mistral-ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/ocr/mistral-ocr.jsonnet new file mode 100644 index 00000000..f70e0cd6 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/ocr/mistral-ocr.jsonnet @@ -0,0 +1,49 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "pdf-decoder" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("mistral-ocr") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-ocr-mistral", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "mistral-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/ocr/ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/ocr/ocr.jsonnet new file mode 100644 index 00000000..4f53aa5a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/ocr/ocr.jsonnet @@ -0,0 +1,37 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "pdf-decoder" +: { + + create:: function(engine) + + local container = + engine.container("pdf-ocr") + .with_image(images.trustgraph_ocr) + .with_command([ + "pdf-ocr", + "-p", + url.pulsar, + ]) + .with_limits("1.0", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "pdf-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/azure-openai.jsonnet new file mode 100644 index 00000000..dcf706d4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/azure-openai.jsonnet @@ -0,0 +1,9 @@ +// Azure OpenAI LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-5.2", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/azure.jsonnet new file mode 100644 index 00000000..6ffa5fdf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/azure.jsonnet @@ -0,0 +1,9 @@ +// Azure LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/bedrock.jsonnet new file mode 100644 index 00000000..6c2c7e90 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/bedrock.jsonnet @@ -0,0 +1,84 @@ +// AWS Bedrock LLM Model Definitions +// Defines available models and their configurations for AWS Bedrock + +{ + "type": "string", + "description": "LLM model to use", + "default": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + "enum": [ + // ── Anthropic Claude ────────────────────────────────────── + { + id: "global.anthropic.claude-opus-4-6-v1", + description: "Claude Opus 4.6 (maximum intelligence)" + }, + { + id: "global.anthropic.claude-opus-4-5-20251101-v1:0", + description: "Claude Opus 4.5 (frontier coding + agents)" + }, + { + id: "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + description: "Claude Sonnet 4.5 (complex agents + coding)" + }, + { + id: "global.anthropic.claude-haiku-4-5-20251001-v1:0", + description: "Claude Haiku 4.5 (fastest with near-frontier intelligence)" + }, + { + id: "global.anthropic.claude-opus-4-1-20250805-v1:0", + description: "Claude Opus 4.1 (agentic search + reasoning)" + }, + { + id: "global.anthropic.claude-sonnet-4-20250514-v1:0", + description: "Claude Sonnet 4.0" + }, + { + id: "anthropic.claude-3-5-haiku-20241022-v1:0", + description: "Claude 3.5 Haiku (legacy)" + }, + + // ── Meta Llama ──────────────────────────────────────────── + { + id: "us.meta.llama4-maverick-17b-instruct-v1:0", + description: "Llama 4 Maverick 17B (128 experts, 400B params, multimodal)" + }, + { + id: "us.meta.llama4-scout-17b-instruct-v1:0", + description: "Llama 4 Scout 17B (16 experts, 3.5M context)" + }, + { + id: "us.meta.llama3-3-70b-instruct-v1:0", + description: "Llama 3.3 70B Instruct" + }, + + // ── Mistral AI ──────────────────────────────────────────── + { + id: "us.mistral.mistral-large-2511-v1:0", + description: "Mistral Large 3 (flagship text, 128K context)" + }, + { + id: "us.mistral.magistral-small-2506-v1:0", + description: "Magistral Small 1.2 (reasoning, cost-effective)" + }, + + // ── DeepSeek ────────────────────────────────────────────── + { + id: "us.deepseek.r1-v1:0", + description: "DeepSeek-R1 (reasoning)" + }, + + // ── Amazon Nova ─────────────────────────────────────────── + { + id: "us.amazon.nova-pro-v1:0", + description: "Amazon Nova Pro (multimodal, balanced)" + }, + { + id: "us.amazon.nova-lite-v1:0", + description: "Amazon Nova Lite (fast, multimodal)" + }, + { + id: "us.amazon.nova-micro-v1:0", + description: "Amazon Nova Micro (text-only, cheapest)" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/chunking-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/chunking-param-types.jsonnet new file mode 100644 index 00000000..f82fa3f7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/chunking-param-types.jsonnet @@ -0,0 +1,25 @@ +// Chunk parameter type definitions + +{ + "chunk-size": { + "type": "integer", + "description": "Chunk size", + "placeholder": 2000, + "helper": "An integer, usually 2000 .. 8000", + "default": 2000, + "min": 0, + "max": 32768, + "required": true + }, + "chunk-overlap": { + "type": "integer", + "description": "Chunk overlap", + "placeholder": 50, + "helper": "An integer, usually 50 .. 100", + "default": 50, + "min": 0, + "max": 8000, + "required": true + }, +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/claude.jsonnet new file mode 100644 index 00000000..b14a317c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/claude.jsonnet @@ -0,0 +1,39 @@ +// Claude LLM Model Definitions +// Defines available models and their configurations for Anthropic's Claude + +{ + "type": "string", + "description": "LLM model to use", + "default": "claude-sonnet-4-5-20250929", + "enum": [ + { + id: "claude-opus-4-6", + description: "Claude Opus 4.6 (maximum intelligence)" + }, + { + id: "claude-opus-4-5-20251101", + description: "Claude Opus 4.5 (frontier coding + agents)" + }, + { + id: "claude-sonnet-4-5-20250929", + description: "Claude Sonnet 4.5 (complex agents + coding)" + }, + { + id: "claude-haiku-4-5-20251001", + description: "Claude Haiku 4.5 (fast)" + }, + { + id: "claude-opus-4-1-20250805", + description: "Claude Opus 4.1 (agentic search + reasoning)" + }, + { + id: "claude-sonnet-4-20250514", + description: "Claude Sonnet 4.0" + }, + { + id: "claude-3-5-haiku-20241022", + description: "Claude 3.5 Haiku" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/cohere.jsonnet new file mode 100644 index 00000000..5a5865a3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/cohere.jsonnet @@ -0,0 +1,35 @@ +// Cohere LLM Model Definitions +// Defines available models and their configurations for Cohere + +{ + "type": "string", + "description": "LLM model to use", + "default": "command-r-plus-08-2024", + "enum": [ + { + id: "command-r-plus-08-2024", + description: "Command R+ (August 2024)" + }, + { + id: "command-r-08-2024", + description: "Command R (August 2024)" + }, + { + id: "command-r-plus", + description: "Command R+ (legacy)" + }, + { + id: "command-r", + description: "Command R (legacy)" + }, + { + id: "command", + description: "Command" + }, + { + id: "command-light", + description: "Command Light" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..477e25dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/embeddings-fastembed.jsonnet @@ -0,0 +1,127 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "sentence-transformers/all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "BAAI/bge-small-en-v1.5", + "description": "bge-small-en-v1.5" + }, + { + "id": "BAAI/bge-small-zh-v1.5", + "description": "bge-small-zh-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-xs", + "description": "snowflake-arctic-embed-xs" + }, + { + "id": "jinaai/jina-embeddings-v2-small-en", + "description": "jina-embeddings-v2-small-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "snowflake/snowflake-arctic-embed-s", + "description": "snowflake-arctic-embed-s" + }, + { + "id": "BAAI/bge-small-en", + "description": "bge-small-en" + }, + { + "id": "BAAI/bge-base-en-v1.5", + "description": "bge-base-en-v1.5" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "description": "paraphrase-multilingual-MiniLM-L12-v2" + }, + { + "id": "Qdrant/clip-ViT-B-32-text", + "description": "clip-ViT-B-32-text" + }, + { + "id": "jinaai/jina-embeddings-v2-base-de", + "description": "jina-embeddings-v2-base-de" + }, + { + "id": "BAAI/bge-base-en", + "description": "bge-base-en" + }, + { + "id": "snowflake/snowflake-arctic-embed-m", + "description": "snowflake-arctic-embed-m" + }, + { + "id": "thenlper/gte-base", + "description": "gte-base" + }, + { + "id": "jinaai/jina-embeddings-v2-base-en", + "description": "jina-embeddings-v2-base-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1", + "description": "nomic-embed-text-v1" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5", + "description": "nomic-embed-text-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-m-long", + "description": "snowflake-arctic-embed-m-long" + }, + { + "id": "jinaai/jina-clip-v1", + "description": "jina-clip-v1" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + { + "id": "jinaai/jina-embeddings-v2-base-es", + "description": "jina-embeddings-v2-base-es" + }, + { + "id": "jinaai/jina-embeddings-v2-base-code", + "description": "jina-embeddings-v2-base-code" + }, + { + "id": "jinaai/jina-embeddings-v2-base-zh", + "description": "jina-embeddings-v2-base-zh" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "description": "paraphrase-multilingual-mpnet-base-v2" + }, + { + "id": "snowflake/snowflake-arctic-embed-l", + "description": "snowflake-arctic-embed-l" + }, + { + "id": "BAAI/bge-large-en-v1.5", + "description": "bge-large-en-v1.5" + }, + { + "id": "thenlper/gte-large", + "description": "gte-large" + }, + { + "id": "intfloat/multilingual-e5-large", + "description": "multilingual-e5-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/embeddings-huggingface.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/embeddings-huggingface.jsonnet new file mode 100644 index 00000000..32254a2b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/embeddings-huggingface.jsonnet @@ -0,0 +1,31 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "all-mpnet-base-v2", + "description": "all-mpnet-base-v2" + }, + { + "id": "all-distilroberta-v1", + "description": "all-distilroberta-v1" + }, + { + "id": "stsb-bert-large", + "description": "stsb-bert-large" + }, + { + "id": "sentence-camembert-large", + "description": "sentence-camembert-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/embeddings-ollama.jsonnet new file mode 100644 index 00000000..c3f29fbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/embeddings-ollama.jsonnet @@ -0,0 +1,23 @@ +// Embeddings model definitions for Ollama +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "all-minilm:latest", + "enum": [ + { + "id": "all-minilm:latest", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/googleaistudio.jsonnet new file mode 100644 index 00000000..6d454ea7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/googleaistudio.jsonnet @@ -0,0 +1,54 @@ +// Google AI Studio LLM Model Definitions +// Defines available models and their configurations for Google AI Studio + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 3 models (preview) + { + id: "gemini-3-pro-preview", + description: "Gemini 3 Pro (preview)" + }, + { + id: "gemini-3-flash-preview", + description: "Gemini 3 Flash (preview)" + }, + + // Gemini 2.5 models + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-001", + description: "Gemini 2.0 Flash" + }, + { + id: "gemini-2.0-flash-lite-001", + description: "Gemini 2.0 Flash Lite" + }, + + // Gemma models + { + id: "gemma-3-27b", + description: "Gemma 3 27B" + }, + { + id: "gemma-3n-e4b", + description: "Gemma 3n E4B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/llamafile.jsonnet new file mode 100644 index 00000000..f6480aa9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/llamafile.jsonnet @@ -0,0 +1,9 @@ +// LlamaFile LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/lmstudio.jsonnet new file mode 100644 index 00000000..1c88da3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/lmstudio.jsonnet @@ -0,0 +1,150 @@ +// LMStudio LLM Model Definitions +// Defines available models and their configurations for LMStudio + +{ + "type": "string", + "description": "LLM model to use", + "default": "llama3.1:70b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/mistral.jsonnet new file mode 100644 index 00000000..07f75b81 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/mistral.jsonnet @@ -0,0 +1,61 @@ +// Mistral LLM Model Definitions +// Defines available models and their configurations for Mistral AI + +{ + "type": "string", + "description": "LLM model to use", + "default": "mistral-medium-2508", + "enum": [ + // Featured models + { + id: "mistral-medium-2508", + description: "Mistral Medium 3.1" + }, + { + id: "mistral-large-2512", + description: "Mistral Large 3" + }, + { + id: "mistral-small-2506", + description: "Mistral Small 3.2" + }, + { + id: "ministral-14b-2512", + description: "Ministral 3 14B" + }, + { + id: "ministral-8b-2512", + description: "Ministral 3 8B" + }, + { + id: "ministral-3b-2512", + description: "Ministral 3 3B" + }, + { + id: "magistral-medium-2509", + description: "Magistral Medium 1.2 (reasoning)" + }, + { + id: "magistral-small-2509", + description: "Magistral Small 1.2 (reasoning)" + }, + { + id: "devstral-2512", + description: "Devstral 2 (code)" + }, + // Other models + { + id: "codestral-2508", + description: "Codestral (code)" + }, + { + id: "pixtral-large-2411", + description: "Pixtral Large (vision)" + }, + { + id: "open-mistral-nemo", + description: "Open Mistral Nemo" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/ollama.jsonnet new file mode 100644 index 00000000..4ef98a6a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/ollama.jsonnet @@ -0,0 +1,150 @@ +// Ollama LLM Model Definitions +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemma3:12b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/openai.jsonnet new file mode 100644 index 00000000..5039c982 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/openai.jsonnet @@ -0,0 +1,51 @@ +// OpenAI LLM Model Definitions +// Defines available models and their configurations for OpenAI's platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-5.2", + "enum": [ + { + id: "gpt-5.2-pro", + description: "GPT-5.2 Pro (maximum intelligence, slow)" + }, + { + id: "gpt-5.2", + description: "GPT-5.2 (flagship, best general-purpose)" + }, + { + id: "gpt-5.2-codex", + description: "GPT-5.2 Codex (optimized for agentic coding)" + }, + { + id: "gpt-5.1", + description: "GPT-5.1 (previous flagship)" + }, + { + id: "gpt-5", + description: "GPT-5 (previous gen reasoning)" + }, + { + id: "gpt-4.1", + description: "GPT-4.1 (coding + long context, 1M tokens)" + }, + { + id: "gpt-4.1-mini", + description: "GPT-4.1 Mini (fast + affordable)" + }, + { + id: "gpt-4.1-nano", + description: "GPT-4.1 Nano (ultra-fast, cheapest)" + }, + { + id: "gpt-4o", + description: "GPT-4o (legacy multimodal)" + }, + { + id: "gpt-4o-mini", + description: "GPT-4o Mini (legacy, budget)" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/temperature-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/temperature-param-types.jsonnet new file mode 100644 index 00000000..f5523e88 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/temperature-param-types.jsonnet @@ -0,0 +1,15 @@ +// LLM temperature definitions + +{ + "llm-temperature": { + "type": "float", + "description": "LLM temperature", + "placeholder": 0.3, + "helper": "A floating point number between 0 and 1", + "default": 0.3, + "min": 0.0, + "max": 10.0, + "required": true + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/vertexai.jsonnet new file mode 100644 index 00000000..34318967 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/vertexai.jsonnet @@ -0,0 +1,81 @@ +// VertexAI LLM Model Definitions +// Defines available models and their configurations for Google's VertexAI platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 3 models (preview) + { + id: "gemini-3-pro-preview", + description: "Gemini 3 Pro (preview)" + }, + { + id: "gemini-3-flash-preview", + description: "Gemini 3 Flash (preview)" + }, + + // Gemini 2.5 models + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-001", + description: "Gemini 2.0 Flash" + }, + { + id: "gemini-2.0-flash-lite-001", + description: "Gemini 2.0 Flash Lite" + }, + + // Gemma models + { + id: "gemma-3-27b", + description: "Gemma 3 27B" + }, + { + id: "gemma-3n-e4b", + description: "Gemma 3n E4B" + }, + + // Claude models on VertexAI + { + id: "claude-opus-4-6", + description: "Claude Opus 4.6" + }, + { + id: "claude-opus-4-5", + description: "Claude Opus 4.5" + }, + { + id: "claude-sonnet-4-5", + description: "Claude Sonnet 4.5" + }, + { + id: "claude-haiku-4-5", + description: "Claude Haiku 4.5" + }, + { + id: "claude-opus-4-1", + description: "Claude Opus 4.1" + }, + { + id: "claude-sonnet-4", + description: "Claude Sonnet 4" + }, + ], + "required": true +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/vllm.jsonnet new file mode 100644 index 00000000..120342b2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/parameters/vllm.jsonnet @@ -0,0 +1,18 @@ +// vLLM Model Definitions +// Defines available models and their configurations for vLLM +// vLLM works with any HuggingFace model. We have to use the model +// vLLM was initialised with + +{ + "type": "string", + "description": "LLM model to use", + "default": "model", + "enum": [ + // Llama 3.1 models + { + id: "model", + description: "Pre-defined model" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/profiles/memory-profile-low.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/profiles/memory-profile-low.jsonnet new file mode 100644 index 00000000..e6341062 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/profiles/memory-profile-low.jsonnet @@ -0,0 +1,169 @@ +// Low memory profile - reduces memory allocation across components. +// Include this at the END of your configuration to override default memory +// settings with lower values suitable for memory-constrained environments. +// +// Usage: {"name": "memory-profile-low", "parameters": {}} +// +// Note: This trades some performance headroom for reduced memory usage. +// Monitor for OOM errors under heavy load. + +{ + + // Override Pulsar stack memory settings + "pulsar" +: { + + // Zookeeper: 512M -> 300M + "zk-memory-limit":: "300M", + "zk-memory-reservation":: "200M", + "zk-heap":: "128m", + "zk-direct-memory":: "64m", + + // Bookie: 1024M -> 600M + "bookie-memory-limit":: "600M", + "bookie-memory-reservation":: "400M", + "bookie-heap":: "128m", + "bookie-direct-memory":: "128m", + + // Broker: 800M -> 512M + "broker-memory-limit":: "512M", + "broker-memory-reservation":: "400M", + "broker-heap":: "192m", + "broker-direct-memory":: "192m", + + // Pulsar-init: 256M -> 128M + "init-memory-limit":: "128M", + "init-memory-reservation":: "128M", + "init-heap":: "64m", + "init-direct-memory":: "64m", + + }, + + // Override Cassandra memory settings: 1000M -> 600M + "cassandra" +: { + "memory-limit":: "600M", + "memory-reservation":: "500M", + "heap":: "200M", + }, + + // Override Qdrant memory settings: 1024M -> 600M + // Also enables mmap for vectors/payloads (trades latency for memory) + "qdrant" +: { + "memory-limit":: "600M", + "memory-reservation":: "500M", + "memmap-threshold-kb":: "1", + "on-disk-payload":: "true", + }, + + // TrustGraph core services - 50% memory reservations + "api-gateway" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "chunker" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "config-svc" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "pdf-decoder" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "mcp-tool" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "mcp-server" +: { + "memory-reservation":: "128M", // 256M -> 128M + }, + + "metering" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "metering-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-store" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-manager" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "prompt" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "prompt-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-embeddings" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "librarian" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "agent-manager" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + // Graph RAG services + "kg-extract-definitions" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-relationships" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-agent" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-ontology" +: { + "memory-reservation":: "150M", // 300M -> 150M + }, + + "kg-extract-objects" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "graph-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "graph-embeddings" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + // Structured data services + "nlp-query" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "structured-query" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "structured-diag" +: { + "memory-reservation":: "48M", // 96M -> 48M + }, + + // Init service + "init-trustgraph" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/agent-kg-extract.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/agent-kg-extract.txt new file mode 100644 index 00000000..36dbd74f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/agent-kg-extract.txt @@ -0,0 +1,28 @@ +Analyze the following text and extract both entity definitions and relationships. + +For definitions, extract entities and their explanations or descriptions. +For relationships, extract subject-predicate-object triples where subjects and objects are entities, and predicates are relationship types. + +Text: {{text}} + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include a "type" field to distinguish between definitions and relationships. + +For definitions, output: +{"type": "definition", "entity": "entity_name", "definition": "definition_text"} + +For relationships, output: +{"type": "relationship", "subject": "subject_entity", "predicate": "relationship_type", "object": "object_entity_or_literal", "object-entity": true} + +Requirements: +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations, only output JSONL + +Example output: +{"type": "definition", "entity": "DNA", "definition": "Deoxyribonucleic acid, a molecule carrying genetic instructions"} +{"type": "definition", "entity": "RNA", "definition": "Ribonucleic acid, essential for coding and gene expression"} +{"type": "relationship", "subject": "DNA", "predicate": "transcribes_to", "object": "RNA", "object-entity": true} +{"type": "relationship", "subject": "DNA", "predicate": "located_in", "object": "cell nucleus", "object-entity": true} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/agent-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/agent-prompt.txt new file mode 100644 index 00000000..9155d20f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/agent-prompt.txt @@ -0,0 +1,104 @@ +# ReAct Agent System Prompt + +You are an AI assistant that uses the ReAct (Reasoning + Acting) framework to solve problems through systematic reasoning and tool use. + +## Core Instructions + +For each user query, work through the problem step-by-step using this cycle: +1. **Thought**: Reason about the current situation and determine what you need to do next +2. **Action**: Take ONE specific action using an available tool +3. Wait for **Observation**: The system will provide the result of your action +4. Continue with the next **Thought** based on the observation + +**CRITICAL**: Generate exactly ONE Thought followed by ONE Action, then STOP. Do not generate multiple Thought/Action pairs in a single response. Do not generate Observations yourself - the system will provide them. + +## Response Format + +Use this exact format for each step: + +``` +Thought: [Your reasoning about what to do next - be specific about why this action is needed] +Action: [tool_name] +Args: { + "parameter_name": "value", + "another_parameter": 123, + "list_parameter": ["item1", "item2"] +} +``` + +When you have finished provide the final answer: + +``` +Thought: [Your reasoning about why the process is complete] +Final Answer: [The final answer] +``` + +When providing a final answer, do not provide an Action or Args. + +## Action Format Rules + +1. **Tool Name**: Write "Action: " followed by the exact tool name on its own line +2. **Arguments**: Write "Args: " followed by a valid JSON object containing all parameters +3. **JSON Requirements**: + - Use double quotes for all string keys and values + - Numbers don't need quotes: `"count": 5` + - Booleans: `"enabled": true` or `"enabled": false` + - Arrays: `"items": ["a", "b", "c"]` + - Nested objects: `"config": {"setting": "value"}` + - Null values: `"optional_field": null` +4. **Required Parameters**: Include all required parameters for the tool +5. **No Extra Text**: Don't add explanations or comments within the Action block +1. **Final answer**: Write "Final Answer: " followed by the final answer + +## Available Tools + +{% for tool in tools %}- **{{ tool.name }}**: {{ tool.description }} +{% for arg in tool.arguments %} - Required: `"{{ arg.name }}"` ({{ arg.type }}): {{ arg.description }} +{% endfor %} +{% endfor %} + +## Behavior Rules + +1. **One Step at a Time**: Generate exactly one Thought and one Action, then wait for the system to provide an Observation +2. **Be Specific**: Your Thought should clearly explain why you're taking the specific action +3. **Use Context**: Build on previous Observations to inform your next steps +4. **Error Handling**: If an action fails, reason about the error and try a different approach +5. **Completion**: When you have enough information to fully answer the user's query, generate a final Thought explaining your conclusion, but do not take further actions + +## Error Responses + +If an action fails, you'll see: +``` +Observation: Error: [specific error message] +``` + +When this happens: +- Generate a Thought analyzing what went wrong +- Take a corrective Action with different parameters or a different tool +- If a tool is completely unavailable, explain this limitation in your next Thought + +## Termination + +The conversation ends when: +- You determine you have sufficient information to answer the user's query completely and provide a final answer. +- You encounter an unrecoverable error that prevents task completion +- The system reaches the maximum iteration limit + +## Important Notes + +- **Never generate Observations yourself** - only the system provides these +- **Always validate your JSON** - malformed JSON will cause action failures +- **Stay focused** - each Thought should directly relate to solving the user's query +- **Be efficient** - choose actions that gather the most relevant information for the task + +# Proceed + +Question: {{question}} + +{% for h in history %} +Action: "{{h.action}}" +Args: { +{% for k, v in h.arguments.items() %} "{{k}}": "{{v}}" +{% endfor %}} +Observation: "{{h.observation}}" +{% endfor %} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/cohere.jsonnet new file mode 100644 index 00000000..9541e4c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/cohere.jsonnet @@ -0,0 +1,42 @@ +// For Cohere. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/default-prompts.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/default-prompts.jsonnet new file mode 100644 index 00000000..fd23d917 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/default-prompts.jsonnet @@ -0,0 +1,285 @@ + +// Prompt templates. For tidy JSONNET use, don't change these templates +// here, but use over-rides in the prompt directory + +{ + + "system-template":: "You are a helpful assistant.", + + "templates":: { + + "question":: { + "prompt": "{{question}}", + }, + + "extract-concepts":: { + "prompt": importstr "extract-concepts.txt", + "response-type": "text", + }, + + "extract-definitions":: { + "prompt": importstr "extract-definitions.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "entity": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "entity", + "definition" + ] + } + }, + + "extract-relationships":: { + "prompt": importstr "extract-relationships.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "object": { + "type": "string" + }, + "object-entity": { + "type": "boolean" + } + }, + "required": [ + "subject", + "predicate", + "object", + "object-entity" + ] + } + }, + + "extract-topics":: { + "prompt": importstr "extract-topics.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "topic": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "topic", + "definition" + ] + } + }, + + "extract-rows":: { + "prompt": importstr "extract-rows.txt", + "response-type": "jsonl", + }, + + "kg-prompt":: { + "prompt": importstr "kg-prompt.txt", + "response-type": "text", + }, + + "kg-edge-reasoning":: { + "prompt": importstr "kg-edge-reasoning.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "reasoning": { + "type": "string" + } + }, + "required": [ + "id", + "reasoning" + ] + } + }, + + "kg-edge-scoring":: { + "prompt": importstr "kg-edge-scoring.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number", + "minimum": 1, + "maximum": 10 + } + }, + "required": [ + "id", + "score" + ] + } + }, + + "kg-synthesis":: { + "prompt": importstr "kg-synthesis.txt", + "response-type": "text", + }, + + "document-prompt":: { + "prompt": importstr "document-prompt.txt", + "response-type": "text", + }, + + "agent-react":: { + "prompt": importstr "agent-prompt.txt", + "response-type": "text" + }, + + "agent-kg-extract":: { + "prompt": importstr "agent-kg-extract.txt", + "response-type": "jsonl", + "object-schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { "const": "definition" }, + "entity": { "type": "string" }, + "definition": { "type": "string" } + }, + "required": ["type", "entity", "definition"] + }, + { + "type": "object", + "properties": { + "type": { "const": "relationship" }, + "subject": { "type": "string" }, + "predicate": { "type": "string" }, + "object": { "type": "string" }, + "object-entity": { "type": "boolean" } + }, + "required": ["type", "subject", "predicate", "object"] + } + ] + } + }, + + "schema-selection":: { + "prompt": importstr "schema-selection.txt", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "An array of schema names that are relevant to answering the given question" + } + }, + + "graphql-generation":: { + "prompt": importstr "graphql-generation.txt", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The GraphQL query string generated to answer the question" + }, + "variables": { + "type": "object", + "description": "Object containing any GraphQL variables needed for the query", + "additionalProperties": true + }, + "confidence": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "description": "Float between 0.0-1.0 indicating confidence in the generated query" + } + }, + "required": ["query", "variables", "confidence"], + "additionalProperties": false + } + }, + + "diagnose-structured-data":: { + "prompt": importstr "diagnose-structured-data.txt", + "response-type": "json", + }, + + "diagnose-xml":: { + "prompt": importstr "diagnose-xml.txt", + "response-type": "json", + }, + "diagnose-json":: { + "prompt": importstr "diagnose-json.txt", + "response-type": "json", + }, + "diagnose-csv":: { + "prompt": importstr "diagnose-csv.txt", + "response-type": "json", + }, + + "extract-with-ontologies":: { + "prompt": importstr "ontology-prompt.txt", + "response-type": "jsonl", + "object-schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { "const": "entity" }, + "entity": { "type": "string" }, + "entity_type": { "type": "string" } + }, + "required": ["type", "entity", "entity_type"] + }, + { + "type": "object", + "properties": { + "type": { "const": "relationship" }, + "subject": { "type": "string" }, + "subject_type": { "type": "string" }, + "relation": { "type": "string" }, + "object": { "type": "string" }, + "object_type": { "type": "string" } + }, + "required": ["type", "subject", "subject_type", "relation", "object", "object_type"] + }, + { + "type": "object", + "properties": { + "type": { "const": "attribute" }, + "entity": { "type": "string" }, + "entity_type": { "type": "string" }, + "attribute": { "type": "string" }, + "value": { "type": "string" } + }, + "required": ["type", "entity", "entity_type", "attribute", "value"] + } + ] + } + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-csv.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-csv.txt new file mode 100644 index 00000000..ad9725ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-csv.txt @@ -0,0 +1,370 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in CSV processing and delimiter-separated value formats. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured CSV data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## CSV Processing Expertise + +When working with CSV data, you must: + +1. **Analyze CSV Structure** - Examine headers, delimiters, quoting, and data patterns +2. **Identify Column Mappings** - Map source column names to target fields +3. **Handle Complex CSV Patterns** - Support various CSV formats including: + - Standard comma-separated values: `name,age,city` + - Alternative delimiters: tab-separated (TSV), pipe-separated, semicolon-separated + - Quoted fields with embedded delimiters: `"Last, First",25,"New York, NY"` + - Headers with spaces or special characters: `"Customer Name","Order Date","Total Amount"` + - Files with or without headers + - Multi-line fields with embedded newlines + +## CSV Format Configuration Guidelines + +For CSV format configurations, use these patterns: + +**Basic CSV Configuration:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "skip_rows": 0 + } + } +} +``` + +**Advanced CSV Options:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": "\t", // Tab-separated + "quote_char": "\"", + "escape_char": "\\", + "has_header": true, + "skip_rows": 2, // Skip metadata rows + "null_values": ["", "NULL", "N/A"], + "trim_whitespace": true, + "skip_blank_lines": true + } + } +} +``` + +**CRITICAL: Source Field Names in Mappings** + +When processing CSV files, the parser uses column headers (if present) or generates column indices as field names. Your source field names in mappings must match exactly: + +**CORRECT Example with Headers:** +CSV file: +```csv +Customer Name,Order Date,Total Amount,Status +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "Customer Name": "John Smith", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Customer Name", // βœ… Correct - matches header exactly + "source_field": "Order Date", // βœ… Correct - matches header exactly + "source_field": "Total Amount", // βœ… Correct - matches header exactly + "source_field": "Status" // βœ… Correct - matches header exactly +} +``` + +**CORRECT Example without Headers:** +CSV file without headers uses column indices: +```csv +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "0": "John Smith", + "1": "2024-01-15", + "2": "1000.50", + "3": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "0", // βœ… Correct - first column + "source_field": "1", // βœ… Correct - second column + "source_field": "2", // βœ… Correct - third column + "source_field": "3" // βœ… Correct - fourth column +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - Delimiter character (comma, tab, pipe, semicolon, etc.) + - Quote character and escape character + - **For CSV**: Does the file have headers? Sample structure + - Text encoding (UTF-8, Windows-1252, etc.) + - Any rows to skip (metadata, blank lines) + - How null/empty values are represented + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source column β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Date/time format conversions + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + - Row-level validation rules + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## CSV Structure Analysis + +When presented with CSV data, analyze: + +1. **Delimiter Detection**: What character separates the fields? +2. **Header Presence**: Does the first row contain column names? +3. **Quote Pattern**: Are fields quoted? What quote character is used? +4. **Data Types**: What types are present in each column? +5. **Null Representation**: How are empty/null values represented? +6. **Special Characters**: Are there embedded commas, quotes, or newlines? +7. **Encoding Issues**: Are there any character encoding problems? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + // CSV-specific parsing options + // delimiter, quote_char, has_header, skip_rows, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `split`, `strip_quotes` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_number`, `parse_currency` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `clean_whitespace`, `normalize_encoding` + +**Date/Time Operations:** +- `parse_date`, `format_date`, `date_component` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `numeric_range`, `date_range` + +## CSV-Specific Best Practices + +1. **Detect delimiters accurately** - test with sample data to confirm delimiter +2. **Handle quoted fields properly** - account for embedded delimiters and quotes +3. **Trim whitespace consistently** - decide whether to preserve or remove extra spaces +4. **Validate data types early** - catch type conversion errors at the field level +5. **Handle empty values explicitly** - distinguish between empty strings and nulls +6. **Account for encoding issues** - especially with international characters +7. **Validate row structure** - ensure consistent column counts across rows + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle CSV-specific edge cases** like malformed quotes, inconsistent delimiters +10. **Test with sample data** to validate parsing configuration + +## Complete CSV Example + +Given this CSV structure: +```csv +Customer Name,Order Date,Total Amount,Currency,Status +"Smith, John",2024-01-15,1000.50,USD,Active +"Doe, Jane",2024-01-16,750.25,EUR,Pending +"Johnson, Bob",2024-01-17,,USD,Cancelled +``` + +The parser will: +1. Use `delimiter: ","` and `quote_char: "\""` to parse fields correctly +2. Use `has_header: true` to treat first row as column names +3. Create this parsed data structure for each record: + ```json + { + "Customer Name": "Smith, John", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Currency": "USD", + "Status": "Active" + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "trim_whitespace": true, + "null_values": ["", "NULL"] + } + }, + "mappings": [ + { + "source_field": "Customer Name", // βœ… Matches header exactly + "target_field": "customer_name", + "transforms": [{"type": "trim"}] + }, + { + "source_field": "Order Date", // βœ… Matches header exactly + "target_field": "order_date", + "transforms": [{"type": "to_date", "format": "YYYY-MM-DD"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "Total Amount", // βœ… Matches header exactly + "target_field": "amount", + "transforms": [ + {"type": "to_float"}, + {"type": "default", "value": 0.0} + ] + }, + { + "source_field": "Currency", // βœ… Matches header exactly + "target_field": "currency_code", + "transforms": [{"type": "upper"}], + "validation": [{"type": "in_list", "values": ["USD", "EUR", "GBP"]}] + }, + { + "source_field": "Status", // βœ… Matches header exactly + "target_field": "order_status", + "transforms": [{"type": "lower"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the column headers exactly, or use column indices (0, 1, 2, etc.) for files without headers.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the CSV structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to delimiter detection, header identification, quoting patterns, and data type inference: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-json.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-json.txt new file mode 100644 index 00000000..2fe6341d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-json.txt @@ -0,0 +1,327 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in JSON processing and JSONPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured JSON data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## JSON Processing Expertise + +When working with JSON data, you must: + +1. **Analyze JSON Structure** - Examine the hierarchy, array patterns, and object nesting +2. **Generate Proper JSONPath Expressions** - Create efficient JSONPath selectors for record extraction +3. **Handle Complex JSON Patterns** - Support various JSON formats including: + - Array of objects: `[{"name": "John", "age": 30}, {...}]` + - Nested object arrays: `{"data": {"records": [{"id": 1}, {...}]}}` + - Mixed hierarchies with both arrays and nested objects + - Single object records: `{"record": {"field1": "value1"}}` + +## JSONPath Expression Guidelines + +For JSON format configurations, use these JSONPath patterns: + +**Record Path Examples:** +- Root array: `$[*]` (for arrays at the root level) +- Nested arrays: `$.data.records[*]` or `$.response.items[*]` +- Single object: `$.record` (when there's one record per file) +- Deep nesting: `$.data.results.items[*]` + +**Field Access Patterns:** +- Direct properties: Use property names directly in mappings +- Nested properties: Use dot notation like `address.street` or `contact.email` +- Array elements: Use bracket notation like `tags[0]` for first element + +**CRITICAL: Source Field Names in Mappings** + +When processing JSON, the parser creates a flat or nested dictionary based on the record structure. Your source field names in mappings must match the actual property names in the parsed records: + +**CORRECT Example:** +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "metadata": { + "source": "UN", + "year": 2024 + } +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches property name + "source_field": "Trade (USD)", // βœ… Correct - matches property name + "source_field": "metadata.source", // βœ… Correct - nested property access + "source_field": "metadata.year" // βœ… Correct - nested property access +} +``` + +**JSON Format Configuration Template:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$[*]", // JSONPath to extract records + "flatten_nested": true, // Whether to flatten nested objects + "array_handling": "expand" // How to handle arrays: expand, first, concat + } + } +} +``` + +**Alternative JSON Options:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.items[*]", // For nested array structures + "flatten_nested": false, // Keep nested structure + "null_value_handling": "skip" // skip, empty_string, or preserve + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - JSON structure type (array of objects, nested objects, single records) + - **For JSON**: Sample structure, nesting patterns, array locations + - Sample data or field descriptions + - Any format-specific details (encoding, special null handling, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Nested object flattening requirements + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or null values + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## JSON Structure Analysis + +When presented with JSON data, analyze: + +1. **Root Structure**: Is it an array, object, or nested structure? +2. **Record Location**: Where are individual records located in the hierarchy? +3. **Field Pattern**: How are field names and values structured? + - Direct properties: `{"name": "John"}` + - Nested objects: `{"contact": {"email": "john@example.com"}}` + - Arrays: `{"tags": ["red", "blue"]}` + - Mixed types: `{"data": [{"id": 1, "details": {"name": "John"}}]}` +4. **Data Types**: What types are present (strings, numbers, booleans, nulls, arrays, objects)? +5. **Hierarchy Depth**: How deeply nested are the records and fields? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + // JSON-specific parsing options + // record_path (JSONPath), flatten_nested, array_handling, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `flatten_object`, `extract_array_element`, `join_array` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## JSON-Specific Best Practices + +1. **Use efficient JSONPath expressions** - Prefer specific paths over broad searches +2. **Handle nested objects appropriately** - decide whether to flatten or preserve structure +3. **Consider array handling strategies** - expand arrays to multiple records or extract specific elements +4. **Account for null vs undefined** values in field mappings +5. **Handle mixed data types** within the same field across records +6. **Use appropriate flattening** for deeply nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle JSON-specific edge cases** like empty arrays, null objects, mixed types + +## Complete JSON Example + +Given this JSON structure: +```json +{ + "data": { + "records": [ + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata": { + "source": "World Bank", + "confidence": 0.95 + } + } + ] + } +} +``` + +The parser will: +1. Use `record_path: "$.data.records[*]"` to extract record objects from the array +2. Create this parsed data structure for each record: + ```json + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata.source": "World Bank", // If flattened + "metadata.confidence": 0.95 // If flattened + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.records[*]", + "flatten_nested": true, + "array_handling": "expand" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches property name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches property name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches property name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + }, + { + "source_field": "metadata.source", // βœ… Flattened nested field + "target_field": "data_source" + } + ] +} +``` + +**KEY RULE: source_field names must match the actual property names in the parsed JSON records, using dot notation for nested properties when flattened.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the JSON structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to JSON hierarchy, object patterns, array structures, and generate appropriate JSONPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-structured-data.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-structured-data.txt new file mode 100644 index 00000000..84c4b8be --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-structured-data.txt @@ -0,0 +1,309 @@ + +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` (will be converted to relative paths automatically) +- With namespaces: `//ns:record` or `//soap:Body/data/record` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name + +**CRITICAL: Source Field Names in Mappings** + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary. Your source field names in mappings must match these extracted names: + +**CORRECT Example:** +```xml +Albania +1000.50 +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50" +} +``` + +So your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)" // βœ… Correct - matches parsed field name +} +``` + +**INCORRECT Example:** +```json +{ + "source_field": "Field[@name='Country or Area']", // ❌ Wrong - XPath not needed here + "source_field": "field[@name='Trade (USD)']" // ❌ Wrong - XPath not needed here +} +``` + +**XML Format Configuration Template:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name" // For value pattern + } + } +} +``` + +**Alternative XML Options:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // Direct element-based records + // No field_attribute needed for standard XML + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - File type (CSV, JSON, XML, Excel, fixed-width, etc.) + - **For XML**: Sample structure, namespace prefixes, record element patterns + - Sample data or field descriptions + - Any format-specific details (delimiters, encoding, namespaces, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Record Container**: Where are individual records located? +3. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns +4. **Namespaces**: Are there any namespace prefixes? +5. **Hierarchy Depth**: How deeply nested are the records? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "[csv|json|xml|fixed-width|excel]", + "encoding": "utf-8", + "options": { + // Format-specific parsing options + // For XML: record_path (XPath), field_attribute (if applicable) + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches +2. **Handle namespace prefixes** when present +3. **Identify field attribute patterns** correctly +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes + +## Complete XML Example + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "/ROOT/data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the extracted field names, NOT the XML element structure.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, and generate appropriate XPath expressions: + +{{sample}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-xml.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-xml.txt new file mode 100644 index 00000000..a3d8efa4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/diagnose-xml.txt @@ -0,0 +1,453 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for XML data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured XML data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + - CDATA sections and text nodes + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` +- With namespaces: `//ns:record` or `//soap:Body/data/record` +- Attribute-filtered: `//record[@type='customer']` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name like `field_attribute: "id"` or `field_attribute: "type"` + +## CRITICAL: Source Field Names in Mappings + +The behavior depends on whether you use `field_attribute`: + +### With field_attribute (Attribute-Based Fields) + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary: + +**Example:** +```xml + + Albania + 1000.50 + Active + +``` + +Parser configuration: +```json +{ + "record_path": "//record", + "field_attribute": "name" +} +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)", // βœ… Correct - matches parsed field name + "source_field": "Status" // βœ… Correct - matches parsed field name +} +``` + +### Without field_attribute (Element-Based Fields) + +When NOT using `field_attribute`, use direct element names or XPath expressions: + +**Example:** +```xml + + Albania + 1000.50 + Active + + UN + 2024 + + +``` + +Parser configuration: +```json +{ + "record_path": "//record" + // No field_attribute specified +} +``` + +Your mappings should use: +```json +{ + "source_field": "country", // βœ… Direct element name + "source_field": "trade_amount", // βœ… Direct element name + "source_field": "metadata/source", // βœ… Nested element path + "source_field": "metadata/year" // βœ… Nested element path +} +``` + +## XML Format Configuration Templates + +**For Attribute-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name", // Extract field names from 'name' attribute + "namespace_prefixes": { // Optional: define namespaces + "ns": "http://example.com/namespace" + } + } + } +} +``` + +**For Element-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // XPath to find record elements + "preserve_namespaces": true, // Optional: keep namespace info + "ignore_attributes": false // Optional: include element attributes + } + } +} +``` + +**For Complex XML with Namespaces:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//soap:Body//data:record", + "namespace_prefixes": { + "soap": "http://schemas.xmlsoap.org/soap/envelope/", + "data": "http://example.com/data" + }, + "field_attribute": "name" + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **XML Structure Details** + - Sample XML structure or schema + - Root element and record location + - Field organization (elements vs attributes) + - Namespace declarations and prefixes + - Text encoding (UTF-8, ISO-8859-1, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Namespaces**: Are there namespace declarations? What prefixes are used? +3. **Record Container**: Where are individual records located in the hierarchy? +4. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns: `John` +5. **Data Location**: Are values in element text, attributes, or both? +6. **Hierarchy Depth**: How deeply nested are the records? +7. **Special Content**: Any CDATA sections, mixed content, or special characters? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + // XML-specific parsing options + // record_path (XPath), field_attribute (if applicable), namespace_prefixes + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `strip_cdata`, `decode_html_entities` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_xml_date`, `parse_iso_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `extract_attribute`, `get_text_content` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `valid_xml_name`, `namespace_check` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches like `//` when possible +2. **Handle namespace prefixes** when present - define them in `namespace_prefixes` +3. **Identify field attribute patterns** correctly - determine if using `field_attribute` +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures +7. **Handle CDATA sections** and special characters properly +8. **Preserve or normalize whitespace** as appropriate +9. **Consider XML schema validation** if XSD is available + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle XML-specific edge cases** like empty elements, mixed content, processing instructions + +## Complete XML Examples + +### Example 1: Attribute-Based Fields + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "//data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +### Example 2: Element-Based Fields + +Given this XML structure: +```xml + + + John Smith + john@example.com +
+ 123 Main St + New York +
+ + + 456 + 100.50 + + +
+
+``` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer" + } + }, + "mappings": [ + { + "source_field": "name", // βœ… Direct element name + "target_field": "customer_name" + }, + { + "source_field": "email", // βœ… Direct element name + "target_field": "email_address" + }, + { + "source_field": "address/street", // βœ… Nested element path + "target_field": "street_address" + }, + { + "source_field": "address/city", // βœ… Nested element path + "target_field": "city" + }, + { + "source_field": "@id", // βœ… Attribute reference + "target_field": "customer_id", + "transforms": [{"type": "to_int"}] + } + ] +} +``` + +**KEY RULES:** +- With `field_attribute`: source_field names must match the extracted attribute values +- Without `field_attribute`: use direct element names, nested paths, or XPath expressions +- Use `@attribute` notation for XML element attributes + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, namespace usage, and generate appropriate XPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/document-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/document-prompt.txt new file mode 100644 index 00000000..5e5d65ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/document-prompt.txt @@ -0,0 +1,7 @@ +Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + +Here is the context: +{{documents}} + +Use only the provided knowledge statements to respond to the following: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-concepts.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-concepts.txt new file mode 100644 index 00000000..cc4b600c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-concepts.txt @@ -0,0 +1,12 @@ +Extract core semantic concepts from the following query. These concepts will be used for semantic search and embedding. + +Output one concept per line. No numbering, no bullet points, no explanation. + +Focus on: +- Key nouns and noun phrases +- Domain-specific terminology +- Languages or time periods mentioned +- Technical terms + +Query: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-definitions.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-definitions.txt new file mode 100644 index 00000000..410ecddf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-definitions.txt @@ -0,0 +1,28 @@ + +Study the following text and derive definitions for any discovered entities. +Do not provide definitions for entities whose definitions are incomplete +or unknown. + +Output each definition as a separate JSON object on its own line (JSONL format). +Each object must have fields: +- entity: the name of the entity +- definition: English text which defines the entity + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not use special characters in the definition text +- Do not include null or unknown definitions + +Example output format: +{"entity": "photosynthesis", "definition": "The process by which plants convert sunlight into energy"} +{"entity": "chlorophyll", "definition": "Green pigment in plants that absorbs light"} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-relationships.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-relationships.txt new file mode 100644 index 00000000..31bcd2d4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-relationships.txt @@ -0,0 +1,28 @@ + +Study the following text and derive entity relationships. For each +relationship, derive the subject, predicate and object of the relationship. + +Output each relationship as a separate JSON object on its own line (JSONL format). +Each object must have fields: +- subject: the subject of the relationship +- predicate: the predicate +- object: the object of the relationship +- object-entity: false if the object is a simple data type (name, value, date). true if it is an entity. + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not use special characters in the text fields + +Example output format: +{"subject": "Earth", "predicate": "orbits", "object": "Sun", "object-entity": true} +{"subject": "Earth", "predicate": "has diameter", "object": "12742 km", "object-entity": false} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-rows.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-rows.txt new file mode 100644 index 00000000..7cea8b7d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-rows.txt @@ -0,0 +1,27 @@ + +Study the following text and derive objects which match the schema provided. + +Output each discovered object as a separate JSON object on its own line (JSONL format). +Each object's fields must carry the name field specified in the schema. + + + +{{schema}} + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object matching the schema +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations + +Example output format (assuming schema has fields: name, age, city): +{"name": "Alice", "age": 30, "city": "London"} +{"name": "Bob", "age": 25, "city": "Paris"} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-topics.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-topics.txt new file mode 100644 index 00000000..bb4b1193 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/extract-topics.txt @@ -0,0 +1,26 @@ +You are a helpful assistant that performs information extraction tasks for a provided text. +Read the provided text. You will identify topics and their definitions. + +Reading Instructions: +- Ignore document formatting in the provided text. +- Study the provided text carefully. + +Here is the text: +{{text}} + +Response Instructions: +- Output each topic as a separate JSON object on its own line (JSONL format) +- Each object must have keys "topic" and "definition" +- Do not respond with special characters +- Return only topics that are concepts and unique to the provided text + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not write any additional text or explanations + +Example output format: +{"topic": "machine learning", "definition": "A subset of AI that enables systems to learn from data"} +{"topic": "neural network", "definition": "A computing system inspired by biological neural networks"} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/gemini.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/gemini.jsonnet new file mode 100644 index 00000000..b9a1e0c0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/gemini.jsonnet @@ -0,0 +1,42 @@ +// For VertexAI Gemini. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/graphql-generation.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/graphql-generation.txt new file mode 100644 index 00000000..58af61dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/graphql-generation.txt @@ -0,0 +1,59 @@ +You are a GraphQL query generation expert. Given a natural language question and provided database schemas, generate a valid GraphQL query embedded in valid JSON. + +## Question: +{{ question }} + +## Provided Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %}- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif %}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif %}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ field.enum_values|join(', ') }}]{% endif %} +{% endfor %} +{% endfor %} + +## GraphQL Query Rules: +1. Use the schema names as GraphQL query fields (e.g., `customers`, `orders`) +2. Apply filters using the `where` parameter with nested filter objects +3. Available filter operators per field type: + - String fields: `eq`, `contains`, `startsWith`, `endsWith`, `in`, `not`, `not_in` + - Integer/Float fields: `eq`, `gt`, `gte`, `lt`, `lte`, `in`, `not`, `not_in` +4. Use `order_by` for sorting (field name as string) +5. Use `direction` for sort direction: `ASC` or `DESC` +6. Use `limit` to restrict number of results +7. Select specific fields in the query body + +## Task Instructions: +1. Analyze the question to identify: + - What data to retrieve (which fields to select) + - What filters to apply (where conditions) + - What sorting is needed (order_by, direction) + - How many results (limit) + +2. Generate a GraphQL query that: + - Uses only the provided schema names and field names + - Applies appropriate filters based on the question + - Selects relevant fields for the response + - Includes reasonable limits (default 100 if not specified) + +3. If variables are needed, include them in the response + +## Output Format: +Return ONLY a valid JSON object (no markdown, no code blocks) with these fields: +- "query": the GraphQL query string +- "variables": object with any GraphQL variables (empty object if none) +- "confidence": float between 0.0-1.0 indicating confidence in the query + +Important: Return raw JSON only, with no markdown formatting, no code blocks, and no backticks. + +## Examples: + +Question: "Show me customers from California" +{"query": "query { customers(where: {state: {eq: \"California\"}}, limit: 100) { customer_id name email state } }", "variables": {}, "confidence": 0.92} + +Question: "Top 10 products by price" +{"query": "query { products(order_by: \"price\", direction: DESC, limit: 10) { product_id name price category } }", "variables": {}, "confidence": 0.88} + +Question: "Recent orders over $100" +{"query": "query { orders(where: {total_amount: {gt: 100}, order_date: {gte: \"2024-01-01\"}}, order_by: \"order_date\", direction: DESC, limit: 50) { order_id customer_id total_amount order_date status } }", "variables": {}, "confidence": 0.96} + +Now generate the GraphQL query for the question above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-edge-reasoning.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-edge-reasoning.txt new file mode 100644 index 00000000..8aeb0c3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-edge-reasoning.txt @@ -0,0 +1,25 @@ +Study the following knowledge statements in Cypher format, extracted from a knowledge graph. Each relationship has an id property. + +Here are the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}} {id: "{{edge.id}}"}]->({{edge.o}}) +{% endfor %} + +For each knowledge statement, explain briefly (5-10 words) why it is relevant to answering the question below. + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include an "id" field and a "reasoning" field. + +{"id": "edge_id", "reasoning": "brief explanation of relevance"} + +Requirements: +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations, only output JSONL + +Example output: +{"id": "a3f9b2c1", "reasoning": "Establishes employment relationship relevant to query"} +{"id": "b7e2d4f9", "reasoning": "Identifies the organisation's sector"} + +Question: {{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-edge-scoring.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-edge-scoring.txt new file mode 100644 index 00000000..40b92773 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-edge-scoring.txt @@ -0,0 +1,25 @@ +Study the following knowledge statements in Cypher format, extracted from a knowledge graph. Each relationship has an id property. + +Here are the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}} {id: "{{edge.id}}"}]->({{edge.o}}) +{% endfor %} + +Score each knowledge statement for relevance to the question below. Assign a score from 1 (not relevant) to 10 (highly relevant). + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include an "id" field and a "score" field. + +{"id": "edge_id", "score": N} + +Requirements: +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations, only output JSONL + +Example output: +{"id": "a3f9b2c1", "score": 9} +{"id": "b7e2d4f9", "score": 3} + +Question: {{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-edge-selection.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-edge-selection.txt new file mode 100644 index 00000000..a8ea44c8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-edge-selection.txt @@ -0,0 +1,26 @@ +Study the following knowledge statements in Cypher format, extracted from a knowledge graph. Each relationship has an id property. + +Here are the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}} {id: "{{edge.id}}"}]->({{edge.o}}) +{% endfor %} + +Select the knowledge statements that are relevant to answering the question below. + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include an "id" field and a "reasoning" field. + +For each relevant edge, output: +{"id": "edge_id", "reasoning": "explanation of why this edge is relevant"} + +Requirements: +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations, only output JSONL + +Example output: +{"id": "a3f9b2c1", "reasoning": "Establishes employment relationship directly relevant to the query"} +{"id": "b7e2d4f9", "reasoning": "Identifies the organisation's sector which contextualises the answer"} + +Question: {{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-prompt.txt new file mode 100644 index 00000000..431c0d73 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-prompt.txt @@ -0,0 +1,8 @@ +Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + +Here's the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}}) +{%endfor%} + +Use only the provided knowledge statements to respond to the following: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-synthesis.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-synthesis.txt new file mode 100644 index 00000000..a80c5594 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/kg-synthesis.txt @@ -0,0 +1,8 @@ +Study the following knowledge statements in Cypher format. Use only these provided statements in your response. Do not speculate if the answer is not found in the provided statements. Answer in natural language. Be helpful and give as complete an answer as possible. + +Here are the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}}) +{% endfor %} + +Use only the provided knowledge statements to respond to the following: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/mixtral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/mixtral.jsonnet new file mode 100644 index 00000000..cd56e7ef --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/mixtral.jsonnet @@ -0,0 +1,42 @@ +// For Mixtral. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/ontology-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/ontology-prompt.txt new file mode 100644 index 00000000..1b451d9c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/ontology-prompt.txt @@ -0,0 +1,82 @@ +You are a knowledge extraction expert. Your task is to find entities, relationships, and attributes in text based on a provided schema. + +## Entity Types + +These are the types of entities you should look for: + +{% for class_id, class_def in classes.items() %} +- **{{class_id}}**{% if class_def.subclass_of %} (subclass of {{class_def.subclass_of}}){% endif %}{% if class_def.comment %}: {{class_def.comment}}{% endif %} +{% endfor %} + +## Relationships + +These relationships connect entities to other entities: + +{% for prop_id, prop_def in object_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Attributes + +These attributes describe entity properties (text, numbers, etc.): + +{% for prop_id, prop_def in datatype_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Text to Analyze + +{{text}} + +## Your Task + +Extract the following from the text above: + +1. **Entities**: Things mentioned in the text and their types +2. **Relationships**: How entities relate to each other +3. **Attributes**: Properties of entities (like quantities, descriptions, etc.) + +## Output Format + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include a "type" field to distinguish between entities, relationships, and attributes. + +For entities, output: +{"type": "entity", "entity": "entity name as it appears in text", "entity_type": "EntityType"} + +For relationships, output: +{"type": "relationship", "subject": "subject entity name", "subject_type": "SubjectType", "relation": "relationship_name", "object": "object entity name", "object_type": "ObjectType"} + +For attributes, output: +{"type": "attribute", "entity": "entity name", "entity_type": "EntityType", "attribute": "attribute_name", "value": "literal value"} + +## Important Rules + +1. **Entity names**: Use the exact text as it appears (e.g., "Cornish pasty", "beef") +2. **Types**: Use the EXACT type identifiers from the schema above (e.g., "fo/Recipe", "fo/Food") +3. **Relationships**: Use the EXACT relationship names from the schema (e.g., "fo/has_ingredient") +4. **Attributes**: Use the EXACT attribute names from the schema (e.g., "fo/serves") +5. **No array brackets**: Output one JSON object per line, not an array +6. **No markdown**: Do not wrap output in code blocks + +## Requirements + +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting, code blocks, or prefixes +- Do not provide explanations, only output JSONL + +## Example + +Input text: "Cornish pasty is a savory pastry filled with beef and potatoes. This recipe serves 4 people." + +Expected output: +{"type": "entity", "entity": "Cornish pasty", "entity_type": "fo/Recipe"} +{"type": "entity", "entity": "beef", "entity_type": "fo/Food"} +{"type": "entity", "entity": "potatoes", "entity_type": "fo/Food"} +{"type": "relationship", "subject": "Cornish pasty", "subject_type": "fo/Recipe", "relation": "fo/has_ingredient", "object": "beef", "object_type": "fo/Food"} +{"type": "relationship", "subject": "Cornish pasty", "subject_type": "fo/Recipe", "relation": "fo/has_ingredient", "object": "potatoes", "object_type": "fo/Food"} +{"type": "attribute", "entity": "Cornish pasty", "entity_type": "fo/Recipe", "attribute": "fo/serves", "value": "4 people"} + +Now extract entities, relationships, and attributes from the text above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/openai.jsonnet new file mode 100644 index 00000000..5d232337 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/openai.jsonnet @@ -0,0 +1,42 @@ +// For OpenAI LLMs. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/schema-selection.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/schema-selection.txt new file mode 100644 index 00000000..39b180e5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/schema-selection.txt @@ -0,0 +1,25 @@ +You are a database schema selection expert. Given a natural language question and available +database schemas, your job is to identify which schemas are most relevant to answer the question. + +## Available Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}): {{ field.description }} +{% endfor %} + +{% endfor %} + +## Question: +{{ question }} + +## Instructions: +1. Analyze the question to understand what data is being requested +2. Examine each schema to understand what data it contains +3. Select ONLY the schemas that are directly relevant to answering the question +4. Return your answer as a JSON array of schema names + +## Response Format: +Return ONLY a JSON array of schema names, nothing else. +Example: ["customers", "orders", "products"] diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/slm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/slm.jsonnet new file mode 100644 index 00000000..48eb96d0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/prompts/slm.jsonnet @@ -0,0 +1,44 @@ +// For SLM. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/pulsar/pulsar-manager.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/pulsar/pulsar-manager.jsonnet new file mode 100644 index 00000000..c1ca4515 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/pulsar/pulsar-manager.jsonnet @@ -0,0 +1,40 @@ +local images = import "values/images.jsonnet"; + +{ + + "pulsar" +: { + + create:: function(engine) + +// FIXME: Should persist something? +// local volume = engine.volume(...) + + local container = + engine.container("pulsar") + .with_image(images.pulsar_manager) + .with_environment({ + SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties", + }) + .with_limits("0.5", "1.4G") + .with_reservations("0.1", "1.4G") + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2"); + + local containerSet = engine.containers( + "pulsar", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2); + + engine.resources([ + containerSet, + service, + ]) + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/pulsar/pulsar.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/pulsar/pulsar.jsonnet new file mode 100644 index 00000000..d417322d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/pulsar/pulsar.jsonnet @@ -0,0 +1,222 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +// This is a Pulsar configuration. Non-standalone mode so we deploy +// individual components: bookkeeper, broker and zookeeper. +// +// This also deploys the TrustGraph 'admin' container which initialises +// TrustGraph-specific namespaces etc. + +{ + + "pulsar" +: { + + // Zookeeper memory settings (can be overridden by memory-profile) + "zk-memory-limit":: "512M", + "zk-memory-reservation":: "512M", + "zk-heap":: "256m", + "zk-direct-memory":: "256m", + + // Bookie memory settings (can be overridden by memory-profile) + "bookie-memory-limit":: "1024M", + "bookie-memory-reservation":: "1024M", + "bookie-heap":: "256m", + "bookie-direct-memory":: "256m", + + // Broker memory settings (can be overridden by memory-profile) + "broker-memory-limit":: "800M", + "broker-memory-reservation":: "800M", + "broker-heap":: "384m", + "broker-direct-memory":: "384m", + + // Pulsar-init memory settings (can be overridden by memory-profile) + "init-memory-limit":: "256M", + "init-memory-reservation":: "256M", + "init-heap":: "128m", + "init-direct-memory":: "128m", + + create:: function(engine) + + // Capture memory settings into locals (self refers to pulsar object here) + local zkMemLimit = self["zk-memory-limit"]; + local zkMemReserv = self["zk-memory-reservation"]; + local zkHeap = self["zk-heap"]; + local zkDirect = self["zk-direct-memory"]; + + local bookieMemLimit = self["bookie-memory-limit"]; + local bookieMemReserv = self["bookie-memory-reservation"]; + local bookieHeap = self["bookie-heap"]; + local bookieDirect = self["bookie-direct-memory"]; + + local brokerMemLimit = self["broker-memory-limit"]; + local brokerMemReserv = self["broker-memory-reservation"]; + local brokerHeap = self["broker-heap"]; + local brokerDirect = self["broker-direct-memory"]; + + local initMemLimit = self["init-memory-limit"]; + local initMemReserv = self["init-memory-reservation"]; + local initHeap = self["init-heap"]; + local initDirect = self["init-direct-memory"]; + + // Zookeeper volume + local zkVolume = engine.volume("zookeeper").with_size("1G"); + + // Zookeeper container + local zkContainer = + engine.container("zookeeper") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper" + ]) + .with_limits("1", zkMemLimit) + .with_reservations("0.05", zkMemReserv) + .with_user("0:1000") + .with_volume_mount(zkVolume, "/pulsar/data/zookeeper") + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + zkHeap, zkHeap, zkDirect, + ], + }) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Pulsar cluster init container + local initContainer = + engine.container("pulsar-init") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650", + ]) + .with_limits("1", initMemLimit) + .with_reservations("0.05", initMemReserv) + .with_environment({ + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + initHeap, initHeap, initDirect, + ], + }); + + + // Bookkeeper volume + local bookieVolume = engine.volume("bookie").with_size("20G"); + + // Bookkeeper container + local bookieContainer = + engine.container("bookie") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + // false ^ causes this to be a 'failure' exit. + ]) + .with_limits("1", bookieMemLimit) + .with_reservations("0.1", bookieMemReserv) + .with_user("0:1000") + .with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper") + .with_environment({ + "clusterName": "cluster-a", + "zkServers": "zookeeper:2181", + "bookieId": "bookie", + "metadataStoreUri": "metadata-store:zk:zookeeper:2181", + "advertisedAddress": "bookie", + "BOOKIE_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + bookieHeap, bookieHeap, bookieDirect, + ], + }) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker, stateless (uses ZK and Bookkeeper for state) + local brokerContainer = + engine.container("pulsar") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" + ]) + .with_limits("1", brokerMemLimit) + .with_reservations("0.1", brokerMemReserv) + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "zookeeperServers": "zookeeper:2181", + "clusterName": "cluster-a", + "managedLedgerDefaultEnsembleSize": "1", + "managedLedgerDefaultWriteQuorum": "1", + "managedLedgerDefaultAckQuorum": "1", + "advertisedAddress": "pulsar", + "advertisedListeners": "external:pulsar://pulsar:6650,localhost:pulsar://localhost:6650", + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + brokerHeap, brokerHeap, brokerDirect, + ], + }) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + // Container sets + local zkContainerSet = engine.containers( + "zookeeper", + [ + zkContainer, + ] + ); + + local initContainerSet = engine.containers( + "init-pulsar", + [ + initContainer, + ] + ); + + local bookieContainerSet = engine.containers( + "bookie", + [ + bookieContainer, + ] + ); + + local brokerContainerSet = engine.containers( + "pulsar", + [ + brokerContainer, + ] + ); + + // Zookeeper service + local zkService = + engine.service(zkContainerSet) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Bookkeeper service + local bookieService = + engine.service(bookieContainerSet) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker service + local brokerService = + engine.service(brokerContainerSet) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + engine.resources([ + zkVolume, + bookieVolume, + zkContainerSet, + initContainerSet, + bookieContainerSet, + brokerContainerSet, + zkService, + bookieService, + brokerService, + ]) + + } + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-additionals.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-additionals.jsonnet new file mode 100644 index 00000000..3e9f3481 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-additionals.jsonnet @@ -0,0 +1,134 @@ +local decode = import "decode-config.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Custom engine that collects configVolume parts +local engine = { + + // Collection of all configVolume parts + configVolumes:: [], + + // Implement all required engine methods as no-ops + container:: function(name) { + with_image:: function(x) self, + with_command:: function(x) self, + with_environment:: function(x) self, + with_limits:: function(c, m) self, + with_reservations:: function(c, m) self, + with_port:: function(src, dest, name) self, + with_volume_mount:: function(vol, mnt) self, + with_user:: function(x) self, + with_runtime:: function(x) self, + with_privileged:: function(x) self, + with_ipc:: function(x) self, + with_capability:: function(x) self, + with_device:: function(hdev, cdev) self, + with_env_var_secrets:: function(vars) self, + }, + + volume:: function(name) { + with_size:: function(size) self, + }, + + // The key method - collects configVolume parts + configVolume:: function(name, dir, parts) + local collector = self + { + configVolumes: super.configVolumes + [ + { + dir: dir, + parts: parts, + } + ] + }; + { + // Return a dummy volume that has the collector in it + name: name, + with_size:: function(size) collector, + // Provide a way to get back to the collector + getCollector:: function() collector, + }, + + secretVolume:: function(name, dir, parts) { + with_size:: function(size) self, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self, + }, + + containers:: function(name, containers) self, + + service:: function(containers) { + with_port:: function(src, dest, name) self, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self, + }, + + resources:: function(res) + // Fold over resources and collect any configVolume state + local collected = std.foldl( + function(state, r) + if std.objectHasAll(r, 'getCollector') then + // Merge the configVolumes from the volume's collector into our state + local volumeCollector = r.getCollector(); + state + { + configVolumes: state.configVolumes + volumeCollector.configVolumes + } + else + state, + res, + self + ); + collected, +}; + +// Execute all component create() functions with our collecting engine +// Note: create:: is a hidden field, so we must use objectHasAll not objectHas +local result = std.foldl( + function(state, p) + if std.objectHasAll(p, 'create') then + // Pattern has create directly - call it + p.create(state) + else + state, + std.objectValues(patterns), + engine +); + +// Debug: show what we collected +local debug = { + numPatterns: std.length(std.objectValues(patterns)), + numConfigVolumes: std.length(result.configVolumes), +}; + +// Transform collected data into output format +local allFiles = std.flattenArrays([ + [ + { + // Remove trailing slash from dir to avoid double slashes + path: std.join("/", [std.rstripChars(cv.dir, "/"), filename]), + content: cv.parts[filename] + } + for filename in std.objectFields(cv.parts) + ] + for cv in result.configVolumes +]); + +// Deduplicate by path - use a map to keep only unique paths +local uniqueMap = std.foldl( + function(acc, item) acc + { [item.path]: item }, + allFiles, + {} +); + +// Convert back to array +local additionals = std.objectValues(uniqueMap); + +// Output the array +additionals diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-aks-k8s.jsonnet new file mode 100644 index 00000000..bcec1cbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-aks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/aks-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-docker-compose.jsonnet new file mode 100644 index 00000000..7f0dbabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-docker-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/docker-compose.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-eks-k8s.jsonnet new file mode 100644 index 00000000..f9de59cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-eks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/eks-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-gcp-k8s.jsonnet new file mode 100644 index 00000000..50037a5c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-gcp-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/gcp-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-minikube-k8s.jsonnet new file mode 100644 index 00000000..6fa64706 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-minikube-k8s.jsonnet @@ -0,0 +1,26 @@ + +local engine = import "../engine/minikube-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +// Extract resources using the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-noop.jsonnet new file mode 100644 index 00000000..3ad735d9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-noop.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/noop.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-ovh-k8s.jsonnet new file mode 100644 index 00000000..92f61041 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-ovh-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/ovh-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-podman-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-podman-compose.jsonnet new file mode 100644 index 00000000..7f0dbabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-podman-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/docker-compose.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-scw-k8s.jsonnet new file mode 100644 index 00000000..4f6c1d7a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-scw-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/scw-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-tg-configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-tg-configuration.jsonnet new file mode 100644 index 00000000..59b4732a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/config-to-tg-configuration.jsonnet @@ -0,0 +1,15 @@ + +local engine = import "../engine/noop.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract configuration directly from patterns +patterns.configuration.configuration + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/decode-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/decode-config.jsonnet new file mode 100644 index 00000000..759513c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/renderers/decode-config.jsonnet @@ -0,0 +1,31 @@ + +local components = import "../components.jsonnet"; + +local apply = function(p, components) + + local base = { + + with:: function(k, v) self + { + [k] +:: v + }, + + with_params:: function(pars) + self + std.foldl( + function(obj, par) obj.with(par.key, par.value), + std.objectKeysValues(pars), + self + ), + + }; + + local component = base + components[p.name]; + + component.with_params(p.parameters); + +local decode = function(config) + local add = function(state, c) state + apply(c, components); + local patterns = std.foldl(add, config, {}); + patterns; + +decode + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/row-store/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/row-store/cassandra.jsonnet new file mode 100644 index 00000000..f90526bc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/row-store/cassandra.jsonnet @@ -0,0 +1,77 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "backends/cassandra.jsonnet"; + +cassandra + { + + "store-rows" +: { + + create:: function(engine) + + local container = + engine.container("store-rows") + .with_image(images.trustgraph_flow) + .with_command([ + "rows-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-rows", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-rows" +: { + + create:: function(engine) + + local container = + engine.container("query-rows") + .with_image(images.trustgraph_flow) + .with_command([ + "rows-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-rows", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/config-composer.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/config-composer.jsonnet new file mode 100644 index 00000000..e93ff044 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/config-composer.jsonnet @@ -0,0 +1,97 @@ +// Configuration Composer Module +// Orchestrates the complete configuration building process +// Combines all components into the final TrustGraph configuration + +local flow_builder = import "flow-builder.jsonnet"; +local interface_builder = import "interface-builder.jsonnet"; + +{ + // Main function to build the complete configuration + build: function(config_spec) + // Extract configuration parameters + local flow_blueprints = config_spec.flow_blueprints; + local default_flow_blueprint = config_spec.default_flow_blueprint; + local default_flow_id = config_spec.default_flow_id; + local flow_init_parameters = config_spec.flow_init_parameters; + + // Build all processors for the default flow + local blueprint_processors = flow_builder.build_blueprint_processors( + flow_blueprints, + default_flow_blueprint, + flow_init_parameters + ); + + local flow_processors = flow_builder.build_flow_processors( + flow_blueprints, + default_flow_blueprint, + default_flow_id, + flow_init_parameters + ); + + // Combine processors into flow objects + local processor_array = blueprint_processors + flow_processors; + local flow_objects = flow_builder.build_flow_objects(processor_array); + local active_flows = flow_builder.merge_flow_objects(flow_objects); + + // Build interfaces for the default flow + local default_flow_interfaces = interface_builder.build_interfaces( + flow_blueprints, + default_flow_blueprint, + default_flow_id, + flow_init_parameters + ); + + // Return object with nested configuration (for backwards compatibility) + { + // Create function (for backwards compatibility) + create: function(engine) {}, + + // The actual configuration object + configuration: { + // Prompts configuration + prompt: { + "system": config_spec.prompts["system-template"], + "template-index": std.objectFieldsAll(config_spec.prompts.templates), + } + { + ["template." + template.key]: template.value + for template in std.objectKeysValuesAll(config_spec.prompts.templates) + }, + + // Tools configuration + tool: { + [tool.id]: tool + for tool in config_spec.tools + }, + + // MCP configuration + mcp: config_spec.mcp, + + // Flow blueprints reference + "flow-blueprint": flow_blueprints, + + // Interface descriptions + "interface-description": config_spec.interface_descriptions, + + // Flow instances + "flow": { + [default_flow_id]: { + "description": "Default processing flow", + "blueprint-name": default_flow_blueprint, + "interfaces": default_flow_interfaces, + "parameters": flow_init_parameters, + }, + }, + + // Active flow processors + "active-flow": active_flows, + + // Token costs and parameter types + "token-cost": config_spec.token_costs, + "parameter-type": config_spec.parameter_types, + + // Collections configuration + "collection": config_spec.collection, + + }, + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/flow-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/flow-builder.jsonnet new file mode 100644 index 00000000..0b142750 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/flow-builder.jsonnet @@ -0,0 +1,72 @@ +// Flow Builder Module +// Processes flow blueprints and builds complete flow configurations +// Handles {blueprint}, {id}, and parameter substitutions + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds blueprint-level processors with parameter substitution + // Processes the 'blueprint' section of flow blueprints + build_blueprint_processors: function(flow_blueprints, blueprint_name, parameters) + [ + [ + // Replace {blueprint} in the processor key + local key = std.strReplace(processor.key, "{blueprint}", blueprint_name); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // First replace {blueprint}, then substitute parameters + local blueprint_replaced = std.strReplace(field.value, "{blueprint}", blueprint_name); + param_processor.substitute_parameters(blueprint_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_blueprints[blueprint_name].blueprint) + ], + + // Builds flow-level processors with parameter substitution + // Processes the 'flow' section of flow blueprints + build_flow_processors: function(flow_blueprints, blueprint_name, flow_id, parameters) + [ + [ + // Replace both {blueprint} and {id} in the processor key + local key = std.strReplace( + std.strReplace(processor.key, "{blueprint}", blueprint_name), + "{id}", flow_id + ); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // Replace {blueprint} and {id}, then substitute parameters + local blueprint_replaced = std.strReplace(field.value, "{blueprint}", blueprint_name); + local id_replaced = std.strReplace(blueprint_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_blueprints[blueprint_name].flow) + ], + + // Combines blueprint and flow processors into flow objects + build_flow_objects: function(processor_array) + std.map( + function(item) { + [item[0][0]] +: { + [item[0][1]]: item[1] + } + }, + processor_array + ), + + // Merges all flow objects into a single flows_active configuration + merge_flow_objects: function(flow_objects) + std.foldr( + function(a, b) a + b, + flow_objects, + {} + ), +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/interface-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/interface-builder.jsonnet new file mode 100644 index 00000000..2143e600 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/interface-builder.jsonnet @@ -0,0 +1,30 @@ +// Interface Builder Module +// Processes flow class interfaces with parameter substitution +// Handles both string interfaces and nested object interfaces + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds interfaces for a specific flow class and instance + // Processes the 'interfaces' section of flow classes + build_interfaces: function(flow_classes, class_name, flow_id, parameters) + local interface_spec = flow_classes[class_name].interfaces; + { + [interface.key]: + if std.isString(interface.value) then + // Simple string interface - apply all substitutions + local class_replaced = std.strReplace(interface.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + else + // Complex object interface - process nested fields + { + [field.key]: + local class_replaced = std.strReplace(field.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(interface.value) + } + for interface in std.objectKeysValuesAll(interface_spec) + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/interface-descriptions.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/interface-descriptions.jsonnet new file mode 100644 index 00000000..f9dda8b0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/interface-descriptions.jsonnet @@ -0,0 +1,89 @@ +// Interface Descriptions Module +// Defines all external interfaces available in TrustGraph flows +// These are the 'endpoints' that external systems can interact with + +{ + // Document loading interfaces - for data ingestion + "document-load": { + "description": "Document loader", + "kind": "send", + "visible": true, + }, + "text-load": { + "description": "Text document loader", + "kind": "send", + "visible": true, + }, + + // Data storage interfaces - for processed data streams + "entity-contexts-load": { + "description": "Entity contexts loader", + "kind": "send", + }, + "triples-store": { + "description": "Triples loader", + "kind": "send", + }, + "graph-embeddings-store": { + "description": "Graph embeddings loader", + "kind": "send", + }, + "document-embeddings-store": { + "description": "Document embeddings loader", + "kind": "send", + }, + "objects-store": { + "description": "Object store", + "kind": "request-response", + }, + + // Query interfaces - for retrieving information + "graph-rag": { + "description": "GraphRAG service", + "kind": "request-response", + }, + "document-rag": { + "description": "ChunkRAG service", + "kind": "request-response", + }, + "triples": { + "description": "Triples query service", + "kind": "request-response", + }, + "graph-embeddings": { + "description": "Graph embeddings service", + "kind": "request-response", + }, + "document-embeddings": { + "description": "Document embeddings service", + "kind": "request-response", + }, + "objects": { + "description": "Object query service", + "kind": "request-response", + }, + + // Processing services - for text and data processing + "prompt": { + "description": "Prompt service", + "kind": "request-response", + }, + "agent": { + "description": "Agent service", + "kind": "request-response", + }, + "text-completion": { + "description": "Text completion service", + "kind": "request-response", + }, + + // Query translation services - for natural language queries + "nlp-query": { + "description": "NLP question to GraphQL service", + "kind": "request-response", + }, + "structured-query": { + "description": "Structured query service", + "kind": "request-response", + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/parameter-processor.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/parameter-processor.jsonnet new file mode 100644 index 00000000..f317ae9f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/parameter-processor.jsonnet @@ -0,0 +1,38 @@ +// Parameter Processing Module +// Handles dynamic parameter replacement in configuration values +// Replaces {parameter_name} placeholders with actual parameter values + +{ + // Applies parameter substitutions to string values + // Only processes strings - leaves other types unchanged + substitute_parameters: function(value, parameters) + if std.isString(value) then + std.foldl( + function(acc, param) + // Only do string replacement if param.value is a string + if std.isString(param.value) then + std.strReplace(acc, "{" + param.key + "}", param.value) + else + acc, // Skip replacement for non-string parameter values + std.objectKeysValuesAll(parameters), + value + ) + else + value, + + // Applies parameter substitutions to all values in an object + // Recursively processes nested objects and arrays + substitute_parameters_in_object: function(obj, parameters) + if std.isObject(obj) then + { + [key]: $.substitute_parameters_in_object(obj[key], parameters) + for key in std.objectFields(obj) + } + else if std.isArray(obj) then + [ + $.substitute_parameters_in_object(item, parameters) + for item in obj + ] + else + $.substitute_parameters(obj, parameters), +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/tools.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/tools.jsonnet new file mode 100644 index 00000000..4947279e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/tools.jsonnet @@ -0,0 +1,36 @@ +// Tools Configuration Module +// Defines all available tools that can be used by agents and flows +// Each tool specifies its interface, arguments, and behavior + +[ + // Knowledge query tool - queries the knowledge base + { + id: "knowledge-query", + name: "Knowledge query", + description: "This tool queries a knowledge base that holds information about domain-specific information. The question should be a natural language question.", + type: "knowledge-query", + collection: "default", + arguments: [ + { + name: "question", + type: "string", + description: "A simple natural language question.", + } + ] + }, + + // LLM completion tool - general purpose text completion + { + id: "llm-completion", + name: "LLM text completion", + type: "text-completion", + description: "This tool queries an LLM for non-domain-specific information. The question should be a natural language question.", + arguments: [ + { + name: "question", + type: "string", + description: "The question which should be asked of the LLM.", + } + ] + } +] \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/trustgraph-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/trustgraph-config.jsonnet new file mode 100644 index 00000000..d860ab3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/runtime-config/trustgraph-config.jsonnet @@ -0,0 +1,90 @@ +// TrustGraph Main Configuration +// Clean, modular composition of TrustGraph configuration +// Uses specialized modules for different aspects of config building + +// Import dependencies +local images = import "../values/images.jsonnet"; +local url = import "../values/url.jsonnet"; +local prompts = import "../prompts/mixtral.jsonnet"; +local default_prompts = import "../prompts/default-prompts.jsonnet"; +local token_costs = import "../values/token-costs.jsonnet"; +local flow_blueprints = import "../flows/flow-blueprints.jsonnet"; +local config_composer = import "config-composer.jsonnet"; +local interface_descriptions = import "interface-descriptions.jsonnet"; +local tools = import "tools.jsonnet"; +local temperature_params = import "../parameters/temperature-param-types.jsonnet"; +local chunking_params = import "../parameters/chunking-param-types.jsonnet"; + +// Main configuration object +local configuration = { + + // Prompt templates + prompts:: default_prompts, + + // Tool definitions + tools:: tools, + + // MCP configuration + mcp:: {}, + + // Flow classes reference + "flow-blueprints":: flow_blueprints, + + // LLM model parameters + "llm-models" +:: {}, + + // Embeddings model parameters + "embeddings-models" +:: {}, + + collections +:: { + "trustgraph:default": { + "user": "default-user", + "collection": "default", + "name": "Default Collection", + "description": "Default collection", + "tags": ["default"], + }, + }, + + // Default model and flow parameters + flow_init_parameters:: { + "llm-model": $["llm-models"].default, + "llm-rag-model": $["llm-models"].default, + "llm-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "llm-rag-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "chunk-size": std.toString($["parameter-types"]["chunk-size"].default), + "chunk-overlap": std.toString($["parameter-types"]["chunk-overlap"].default), + "embeddings-model": $["embeddings-models"].default, + }, + + // Interface descriptions for external endpoints + "interface-descriptions":: interface_descriptions, + + // Parameter type definitions + "parameter-types":: { + "llm-model": $["llm-models"], + "embeddings-model": $["embeddings-models"], + } + chunking_params + temperature_params, + + // Token costs + "token-costs":: token_costs, + + // Build the complete configuration using the composer + configuration:: config_composer.build({ + flow_blueprints: $["flow-blueprints"], + default_flow_blueprint: "everything", + default_flow_id: "default", + flow_init_parameters: $["flow_init_parameters"], + prompts: $["prompts"], + tools: $["tools"], + mcp: $["mcp"], + interface_descriptions: $["interface-descriptions"], + parameter_types: $["parameter-types"], + token_costs: $["token-costs"], + collection: $["collections"], + }), + +} + default_prompts; + +// Export the final configuration +configuration diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/cassandra.jsonnet new file mode 100644 index 00000000..05f4fd27 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/cassandra.jsonnet @@ -0,0 +1,77 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "backends/cassandra.jsonnet"; + +cassandra + { + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-cassandra", + "-p", + url.pulsar, + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/falkordb.jsonnet new file mode 100644 index 00000000..b4bace2d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/falkordb.jsonnet @@ -0,0 +1,79 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local falkordb = import "backends/falkordb.jsonnet"; + +falkordb + { + + "falkordb-url":: "falkor://falkordb:6379", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-falkordb", + "-p", + url.pulsar, + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/memgraph.jsonnet new file mode 100644 index 00000000..72597d72 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/memgraph.jsonnet @@ -0,0 +1,84 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local memgraph = import "backends/memgraph.jsonnet"; + +memgraph + { + + "memgraph-url":: "bolt://memgraph:7687", + "memgraph-database":: "memgraph", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-memgraph", + "-p", + url.pulsar, + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/neo4j.jsonnet new file mode 100644 index 00000000..08904715 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/triple-store/neo4j.jsonnet @@ -0,0 +1,79 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local neo4j = import "backends/neo4j.jsonnet"; + +neo4j + { + + "neo4j-url":: "bolt://neo4j:7687", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-neo4j", + "-p", + url.pulsar, + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/ui/workbench-ui.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/ui/workbench-ui.jsonnet new file mode 100644 index 00000000..f2048e47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/ui/workbench-ui.jsonnet @@ -0,0 +1,32 @@ +local images = import "values/images.jsonnet"; + +{ + + "workbench-ui" +: { + + create:: function(engine) + + local container = + engine.container("workbench-ui") + .with_image(images["workbench-ui"]) + .with_limits("0.1", "256M") + .with_reservations("0.1", "256M") + .with_port(8888, 8888, "ui"); + + local containerSet = engine.containers( + "workbench-ui", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8888, 8888, "ui"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/values/images.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/values/images.jsonnet new file mode 100644 index 00000000..489859bc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/values/images.jsonnet @@ -0,0 +1,36 @@ +local version = import "version.jsonnet"; +{ + cassandra: "docker.io/cassandra:5.0.6", +// Not working +// ceph: "quay.io/ceph/daemon:latest-reef", + neo4j: "docker.io/neo4j:2025.08.0-community-bullseye", + pulsar: "docker.io/apachepulsar/pulsar:4.1.0", + pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.4.0", + etcd: "quay.io/coreos/etcd:v3.6.4", + minio: "docker.io/minio/minio:RELEASE.2025-09-07T16-13-09Z", + garage: "docker.io/dxflrs/garage:v2.1.0", + milvus: "docker.io/milvusdb/milvus:v2.5.17", + prometheus: "docker.io/prom/prometheus:v3.8.0", + grafana: "docker.io/grafana/grafana:12.3.0", + loki: "docker.io/grafana/loki:3.6.2", + trustgraph_base: "docker.io/trustgraph/trustgraph-base:" + version, + trustgraph_flow: "docker.io/trustgraph/trustgraph-flow:" + version, + trustgraph_ocr: "docker.io/trustgraph/trustgraph-ocr:" + version, + trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version, + trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version, + trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version, + trustgraph_mcp: "docker.io/trustgraph/trustgraph-mcp:" + version, + qdrant: "docker.io/qdrant/qdrant:v1.15.4", + memgraph_mage: "docker.io/memgraph/memgraph-mage:3.5", + memgraph_lab: "docker.io/memgraph/lab:3.5.0", + falkordb: "docker.io/falkordb/falkordb:v4.12.5", + "workbench-ui": "docker.io/trustgraph/workbench-ui:1.7.2", + "ddg-mcp-server": "docker.io/trustgraph/ddg-mcp-server:0.1.0", + "tgi-service-intel-xpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-xpu", + "tgi-service-cpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-cpu", + "tgi-service-gaudi": "ghcr.io/huggingface/text-generation-inference:sha-f140440-gaudi", + "vllm-service-intel-xpu": "docker.io/intel/vllm:0.8.0-xpu", + "vllm-service-gaudi": "docker.io/trustgraph/vllm-hpu:027f5645", + "vllm-service-nvidia": "docker.io/vllm/vllm-openai:latest", + "vllm-service-intel-battlemage": "docker.io/intelanalytics/ipex-llm-serving-xpu:0.2.0-b6", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/values/token-costs.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/values/token-costs.jsonnet new file mode 100644 index 00000000..84c70373 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/values/token-costs.jsonnet @@ -0,0 +1,102 @@ +{ + "mistral.mistral-large-2407-v1:0": { + "model_name": "mistral.mistral-large-2407-v1:0", + "input_price": 0.000004, + "output_price": 0.000012 + }, + "meta.llama3-1-405b-instruct-v1:0": { + "model_name": "meta.llama3-1-405b-instruct-v1:0", + "input_price": 0.00000532, + "output_price": 0.000016 + }, + "mistral.mixtral-8x7b-instruct-v0:1": { + "model_name": "mistral.mixtral-8x7b-instruct-v0:1", + "input_price": 0.00000045, + "output_price": 0.0000007 + }, + "meta.llama3-1-70b-instruct-v1:0": { + "model_name": "meta.llama3-1-70b-instruct-v1:0", + "input_price": 0.00000099, + "output_price": 0.00000099 + }, + "meta.llama3-1-8b-instruct-v1:0": { + "model_name": "meta.llama3-1-8b-instruct-v1:0", + "input_price": 0.00000022, + "output_price": 0.00000022 + }, + "anthropic.claude-3-haiku-20240307-v1:0": { + "model_name": "anthropic.claude-3-haiku-20240307-v1:0", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "anthropic.claude-3-5-sonnet-20240620-v1:0": { + "model_name": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "cohere.command-r-plus-v1:0": { + "model_name": "cohere.command-r-plus-v1:0", + "input_price": 0.0000030, + "output_price": 0.0000150 + }, + "ollama": { + "model_name": "ollama", + "input_price": 0, + "output_price": 0 + }, + "claude-3-haiku-20240307": { + "model_name": "claude-3-haiku-20240307", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "claude-3-5-sonnet-20240620": { + "model_name": "claude-3-5-sonnet-20240620", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "claude-3-opus-20240229": { + "model_name": "claude-3-opus-20240229", + "input_price": 0.000015, + "output_price": 0.000075 + }, + "claude-3-sonnet-20240229": { + "model_name": "claude-3-sonnet-20240229", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "command-r-08-202": { + "model_name": "command-r-08-202", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "c4ai-aya-23-8b": { + "model_name": "c4ai-aya-23-8b", + "input_price": 0, + "output_price": 0 + }, + "llama.cpp": { + "model_name": "llama.cpp", + "input_price": 0, + "output_price": 0 + }, + "gpt-4o": { + "model_name": "gpt-4o", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-2024-08-06": { + "model_name": "gpt-4o-2024-08-06", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "gpt-4o-2024-05-13": { + "model_name": "gpt-4o-2024-05-13", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-mini": { + "model_name": "gpt-4o-mini", + "input_price": 0.00000015, + "output_price": 0.0000006 + } +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/values/url.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/values/url.jsonnet new file mode 100644 index 00000000..c3d4ad97 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/values/url.jsonnet @@ -0,0 +1,7 @@ +{ + pulsar: "pulsar://pulsar:6650", + pulsar_admin: "http://pulsar:8080", + milvus: "http://milvus:19530", + qdrant: "http://qdrant:6333", + object_store: "garage:3900", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/vector-store/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/vector-store/milvus.jsonnet new file mode 100644 index 00000000..213027d1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/vector-store/milvus.jsonnet @@ -0,0 +1,146 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local milvus = import "backends/milvus.jsonnet"; + +milvus + { + + "store-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-milvus", + "-p", + url.pulsar, + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/vector-store/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/vector-store/pinecone.jsonnet new file mode 100644 index 00000000..3803bdcb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/vector-store/pinecone.jsonnet @@ -0,0 +1,160 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; + +{ + + "pinecone-cloud":: "aws", + "pinecone-region":: "us-east-1", + + "store-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-pinecone", + "-p", + url.pulsar, + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/vector-store/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/vector-store/qdrant.jsonnet new file mode 100644 index 00000000..06c9f0e1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/vector-store/qdrant.jsonnet @@ -0,0 +1,250 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local qdrant = import "backends/qdrant.jsonnet"; + +qdrant + { + + "store-graph-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "doc-embeddings-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "doc-embeddings-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + }, + + "store-row-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-row-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "row-embeddings-write-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-row-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-row-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-row-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "row-embeddings-query-qdrant", + "-p", + url.pulsar, + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-row-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/zip-readme.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/zip-readme.md new file mode 100644 index 00000000..0b117792 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.1/zip-readme.md @@ -0,0 +1,28 @@ + +Note! this is a subset of possible configurations, to generate your own +launch config use the config util... + +- Production: https://config-ui.demo.trustgraph.ai +- Early release: https://dev.config-ui.demo.trustgraph.ai + +The config util auto-generates deployment instructions for your +configuration, so that's the recommended way to deploy. + +---------------------------------------------------------------------------- + +These are launch configurations for TrustGraph. See https://trustgraph.ai for +the quickstart using docker compose. + +Hint for Linux: There are files here which get mounted as volumes inside +Docker Compose containers. This may trigger SELinux rules on your system, to +permit access insider the containers, use a command like this... + +chcon -Rt svirt_sandbox_file_t grafana/ prometheus/ + +The file vertexai/private.json is a placeholder for real GCP credentials if +you are using the VertexAI LLM. If you're using that in Docker Compose, +replace with your real credentials, and don't forget to permit access if you +are using Linux: + +chcon -Rt svirt_sandbox_file_t vertexai/ + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/README.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/README.md new file mode 100644 index 00000000..23039e9a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/README.md @@ -0,0 +1,125 @@ + +# TrustGraph template generation + +There are two utilities here: + +- `generate`: Generates a single Docker Compose launch configuration + based on configuration you provide. +- `generate-all`: Generates the release bundle for releases. You won't + need to use this unless you are managing releases. + +## `generate-all` + +Previously, this generates a full set of all vector DB / triple store / LLM +combinations, and put them in a single ZIP file. But this got out of +hand, so at the time of writing, this generates a single configuraton +using Qdrant vector DB, Ollama LLM support and Cassandra for a triple store. + +The combinations are contained withing the code, it takes two arguments: +- output ZIP file (is over-written) +- TrustGraph version number + +``` +templates/generate-all output.zip 0.18.11 +``` + +## `generate` + +This utility takes a configuration file describing the components to bundle, +and outputs a Docker Compose YAML file. + +### Input configuration + +The input configuration is a JSON file, an array of components to pull into +the configuration. For each component, there is a name and a (possibly empty) +object describing addtional parameters for that component. + +Example: + +``` +[ + { + "name": "cassandra", + "parameters": {} + }, + { + "name": "pulsar", + "parameters": {} + }, + { + "name": "qdrant", + "parameters": {} + }, + { + "name": "embeddings-hf", + "parameters": {} + }, + { + "name": "graph-rag", + "parameters": {} + }, + { + "name": "grafana", + "parameters": {} + }, + { + "name": "trustgraph", + "parameters": {} + }, + { + "name": "googleaistudio", + "parameters": { + "googleaistudio-temperature": 0.3, + "googleaistudio-max-output-tokens": 2048, + "googleaistudio-model": "gemini-1.5-pro-002" + } + }, + { + "name": "prompt-template", + "parameters": {} + }, + { + "name": "override-recursive-chunker", + "parameters": { + "chunk-size": 1000, + "chunk-overlap": 50 + } + }, + { + "name": "workbench-ui", + "parameters": {} + }, + { + "name": "agent-manager-react", + "parameters": {} + } +] +``` + +If you want to make your own configuration you could try changing the +configuration above: +- Components which are essential: pulsar, trustgraph, graph-rag, grafana, + agent-manager-react +- You need a triple store, one of: cassandra, memgraph, falkordb, neo4j +- You need a vector store, one of: qdrant, pinecone +- You need an LLM, one of: azure, azure-openai, bedrock, claude, cohere, + llamafile, ollama, openai, vertexai. +- You need an embeddings implementation, one of: embeddings-hf, + embeddings-ollama +- Optionally add the Workbench tool: workbench-ui + +Components have over-ridable parameters, look in the component definition +in `templates/components/` to see what you can override. + +### Invocation + +Two parameters: +- The output ZIP file +- The version number + +The configuration file described above is provided on standard input + +``` +templates/generate out.zip 0.18.9 < config.json +``` + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/cassandra.jsonnet new file mode 100644 index 00000000..60f262d3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/cassandra.jsonnet @@ -0,0 +1,50 @@ +local images = import "values/images.jsonnet"; + +{ + + "cassandra" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1000M", + "memory-reservation":: "1000M", + "heap":: "300M", + + create:: function(engine) + + // Capture memory settings into locals + local memLimit = self["memory-limit"]; + local memReserv = self["memory-reservation"]; + local heap = self["heap"]; + + local vol = engine.volume("cassandra").with_size("20G"); + + local container = + engine.container("cassandra") + .with_image(images.cassandra) + .with_environment({ + JVM_OPTS: "-Xms%s -Xmx%s -Dcassandra.skip_wait_for_gossip_to_settle=0" % [ + heap, heap, + ], + }) + .with_limits("1.0", memLimit) + .with_reservations("0.5", memReserv) + .with_port(9042, 9042, "cassandra") + .with_volume_mount(vol, "/var/lib/cassandra"); + + local containerSet = engine.containers( + "cassandra", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9042, 9042, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/ceph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/ceph.jsonnet new file mode 100644 index 00000000..42406a95 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/ceph.jsonnet @@ -0,0 +1,446 @@ + +// +// This majorly does not work +// + +// This configuration fails primarily because it tries to treat Ceph +// like a stateless web app. You are currently pointing mon host to a +// generic service name (ceph-mon), but you aren't telling the Monitor +// process to assume that service identity. + +// To make this work with the "Service Name" approach across any +// engine, you need to fix the binding logic and the messenger protocol. + +// 1. The ceph.conf Fix + +// Need to enable Messenger v2 (modern) and tell the cluster to use +// the service names for its initial quorum. + +//Change this section: +// Ini, TOML + +// [global] +// fsid = %s +// mon initial members = mon0 +// mon host = ceph-mon:6789 +// ... + +// To this: +// Ini, TOML + +// [global] +// fsid = %s +// # Use the actual service names as members +// mon initial members = ceph-mon +// # Explicitly use the Service name (VIP) +// mon host = ceph-mon +// # Force modern protocol +// ms_bind_msgr2 = true +// ms_bind_msgr1 = true + +// 2. The MON Environment & Command Fix + +// This is the most critical part. Your current config sets MON_IP: +// "0.0.0.0". This causes the MON to bind to the Pod IP, which breaks +// when the Pod restarts. You must force it to bind to the Service IP. + +// Update mon_env: +// Code snippet + +// local mon_env = cluster_env + { +// CEPH_DAEMON: "MON", +// MON_NAME: "mon0", +// # Remove MON_IP: "0.0.0.0" +// # Add these: +// MON_ADDR: "ceph-mon", // This says to resolve the service name +// }; + +// Update mon_container command: You are currently wiping the MON data +// on every start (rm -rf /var/lib/ceph/mon/*). Stop doing that. If you +// wipe the data, you lose the cluster state and the OSDs will refuse to +// talk to the "new" MON. + +// Code snippet + +// .with_command([ +// "bash", "-c", +// # 1. Resolve the Service IP at runtime +// "export MON_IP=$(getent hosts ceph-mon | awk '{ print $1 }'); " + +// inject_mon_config + +// # 2. Start the daemon telling it its PUBLIC address is the Service VIP +// "exec /opt/ceph-container/bin/entrypoint.sh" +// ]) + +// 3. Why your current config "Majorly does not work" + +// The "Wipe" Logic: By running rm -rf /var/lib/ceph/mon/* in the +// MON container, you are creating a "New Cluster" every time the +// container starts. Since the OSDs store the fsid and cluster +// secrets, they will reject the "new" MON. + +// DNS Race Condition: Your OSD/MGR/RGW containers wait for +// ceph-mon DNS, which is good. However, if ceph-mon resolves to a +// Round Robin IP (multiple pods) rather than a stable ClusterIP, the +// connection will be flaky. + +// Messenger Protocol: Without ms_bind_msgr2, Ceph defaults to the +// old v1 protocol which is much more sensitive to NAT/Container IP +// mismatches. + +// SUMMARY + +// Component: ceph.conf +// Change: Add ms_bind_msgr2 = true +// Why: Supports modern container networking better. + +// Component: MON Start +// Change: Remove rm -rf +// Why: Ceph MONs must keep their database to maintain the cluster. + +// Component: MON Address +// Change: Use getent hosts ceph-mon +// Why: Forces the MON to advertise the Service VIP instead of its own Pod IP. + +// Component: MON Keyring +// Change: Ensure /etc/ceph/ceph.mon.keyring exists +// Why: MONs need their specific key to start. + +local images = import "values/images.jsonnet"; + +{ + with:: function(key, value) + self + { + ["ceph-" + key]:: value, + }, + + // Ceph credentials and cluster settings + "ceph-access-key":: "object-user", + "ceph-secret-key":: "object-password", + "ceph-cluster-id":: "ceph", + "ceph-fsid":: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", + + // Pool redundancy settings + // size: 2 = two replicas for fault tolerance + // min_size: 1 = allow degraded I/O if one OSD is down (prevents cluster freeze) + "ceph-pool-size":: "2", + "ceph-pool-min-size":: "1", + + ceph +: { + create:: function(engine) + // Pre-Shared Cryptographic Material - Config-as-Code Approach + // These keys are generated once and distributed to all daemons + // This ensures cryptographic consistency across the shared-nothing architecture + local admin_key = "AQBpxSBlAAAAABAAU99V6D8vS7Uu9y1S8W0iBg=="; + local mon_key = "AQBpxSBlAAAAABAAn7pL/pG9oT+X6vO7V1S6bg=="; + + // Ceph configuration file - rendered from Jsonnet variables + local ceph_conf = ||| + [global] + fsid = %s + mon initial members = mon0 + mon host = ceph-mon:6789 + public network = 0.0.0.0/0 + cluster network = 0.0.0.0/0 + osd pool default size = %s + osd pool default min size = %s + osd crush chooseleaf type = 0 + auth cluster required = cephx + auth service required = cephx + auth client required = cephx + ||| % [$["ceph-fsid"], $["ceph-pool-size"], $["ceph-pool-min-size"]]; + + // Admin keyring - distributed to all daemons + local admin_keyring = ||| + [client.admin] + key = %s + caps mds = "allow *" + caps mgr = "allow *" + caps mon = "allow *" + caps osd = "allow *" + ||| % [admin_key]; + + // Monitor keyring - used by MON for cluster operations + local mon_keyring = ||| + [mon.] + key = %s + caps mon = "allow *" + ||| % [mon_key]; + + // Config injection command - writes files before entrypoint + local inject_config = "printf '%s' > /etc/ceph/ceph.conf; printf '%s' > /etc/ceph/ceph.client.admin.keyring; " % [ceph_conf, admin_keyring]; + local inject_mon_config = inject_config + ("printf '%s' > /etc/ceph/ceph.mon.keyring; " % [mon_keyring]); + + // Data volumes - sized appropriately for production workloads + local vol_mon = engine.volume("ceph-mon").with_size("20G"); + local vol_mgr = engine.volume("ceph-mgr").with_size("20G"); + local vol_osd = engine.volume("ceph-osd").with_size("100G"); + local vol_rgw = engine.volume("ceph-rgw").with_size("20G"); + + // Isolated config volumes per daemon (ReadWriteOnce compatible) + // Each daemon gets its own non-shared config volume to support + // multi-node scheduling in K8s and other orchestrators + local vol_mon_config = engine.volume("ceph-mon-config").with_size("500M"); + local vol_mgr_config = engine.volume("ceph-mgr-config").with_size("500M"); + local vol_osd_config = engine.volume("ceph-osd-config").with_size("500M"); + local vol_rgw_config = engine.volume("ceph-rgw-config").with_size("500M"); + local vol_init_config = engine.volume("ceph-init-config").with_size("500M"); + + // Simplified cluster environment - Config-as-Code model + // No fetch logic needed - config is injected before entrypoint runs + local cluster_env = { + CLUSTER: $["ceph-cluster-id"], + FSID: $["ceph-fsid"], + KV_TYPE: "none", // No external coordination + }; + + // MON-specific environment + // Config-as-Code: MON uses injected config files, not fetch logic + // + // CRITICAL: MON_DATA_AVAIL="0" forces fresh cluster bootstrap + // The ceph/daemon entrypoint script (variables_stack.sh) uses this as a gate: + // - MON_DATA_AVAIL="0" -> run mkfs, create new cluster with our FSID + // - MON_DATA_AVAIL="1" -> attempt to join existing cluster (infinite probe loop) + // + // Network configuration for monmap generation + local mon_env = cluster_env + { + CEPH_DAEMON: "MON", + MON_NAME: "mon0", + MON_PORT: "6789", + MON_DATA_AVAIL: "0", + MON_IP: "0.0.0.0", + NETWORK_AUTO_DETECT: "4", + CEPH_PUBLIC_NETWORK: "0.0.0.0/0", + }; + + // Simplified daemon environments - Config-as-Code model + // All daemons receive config via injection, not fetch from MON + // This eliminates "static mode" errors and networking complexity + + // MGR-specific environment + local mgr_env = cluster_env + { + CEPH_DAEMON: "MGR", + MGR_NAME: "mgr0", + }; + + // OSD-specific environment + local osd_env = cluster_env + { + CEPH_DAEMON: "OSD", + OSD_TYPE: "directory", + }; + + // RGW-specific environment + local rgw_env = cluster_env + { + CEPH_DAEMON: "RGW", + RGW_NAME: "rgw0", + RGW_FRONTEND_PORT: "7480", + }; + + // MON (Monitor) container - cluster state and quorum + // Config-as-Code: Injects pre-shared keys before entrypoint + // CRITICAL: Wipes /var/lib/ceph/mon/* on every start to force fresh bootstrap + // This ensures MON always uses our FSID and doesn't inherit stale cluster state + local mon_container = + engine.container("ceph-mon") + .with_image(images.ceph) + .with_environment(mon_env) + .with_command([ + "bash", "-c", + "rm -rf /var/lib/ceph/mon/*; " + + inject_mon_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(6789, 6789, "mon") + .with_port(3300, 3300, "mon-msgr2") + .with_volume_mount(vol_mon, "/var/lib/ceph/mon") + .with_volume_mount(vol_mon_config, "/etc/ceph"); + + // MGR (Manager) container - cluster management and dashboard + // Config-as-Code: Uses injected config files with pre-shared keys + // DNS wait ensures MON is available before MGR connects + local mgr_container = + engine.container("ceph-mgr") + .with_image(images.ceph) + .with_environment(mgr_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(7000, 7000, "mgr") + .with_port(8443, 8443, "dashboard") + .with_port(9283, 9283, "prometheus") + .with_volume_mount(vol_mgr, "/var/lib/ceph/mgr") + .with_volume_mount(vol_mgr_config, "/etc/ceph"); + + // OSD (Object Storage Daemon) - actual data storage + // Config-as-Code: Uses injected config files with pre-shared keys + // Increased resources to prevent OOM during recovery operations + // DNS wait ensures MON is available before OSD connects + local osd_container = + engine.container("ceph-osd") + .with_image(images.ceph) + .with_environment(osd_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("2.0", "4096M") + .with_reservations("1.0", "2048M") + .with_port(6800, 6800, "osd") + .with_volume_mount(vol_osd, "/var/lib/ceph/osd") + .with_volume_mount(vol_osd_config, "/etc/ceph"); + + // RGW (RADOS Gateway) - S3 API endpoint + // Config-as-Code: Uses injected config files with pre-shared keys + // DNS wait ensures MON is available before RGW connects + local rgw_container = + engine.container("ceph-rgw") + .with_image(images.ceph) + .with_environment(rgw_env) + .with_command([ + "bash", "-c", + "until getent hosts ceph-mon; do echo 'Waiting for MON DNS...'; sleep 2; done; " + + inject_config + + "exec /opt/ceph-container/bin/entrypoint.sh" + ]) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1024M") + .with_port(7480, 7480, "s3") + .with_volume_mount(vol_rgw, "/var/lib/ceph/radosgw") + .with_volume_mount(vol_rgw_config, "/etc/ceph"); + + // Init container - one-time S3 user provisioning + // IMPORTANT: This container exits with code 0 after completion + // Orchestrator must NOT restart it (use K8s Job or Compose restart: "no") + // Config-as-Code: Uses injected config to run radosgw-admin commands + local init_container = + engine.container("ceph-init") + .with_image(images.ceph) + .with_environment({ + CLUSTER: $["ceph-cluster-id"], + FSID: $["ceph-fsid"], + KV_TYPE: "none", + RGW_ACCESS_KEY: $["ceph-access-key"], + RGW_SECRET_KEY: $["ceph-secret-key"], + }) + .with_limits("0.5", "512M") + .with_reservations("0.25", "256M") + .with_volume_mount(vol_init_config, "/etc/ceph") + .with_command([ + "bash", "-c", + inject_config + ||| + set -e + + # Wait for cluster health + echo "Waiting for Ceph cluster to be healthy..." + MAX_ATTEMPTS=60 + ATTEMPT=0 + until ceph --cluster ${CLUSTER} health 2>/dev/null | grep -q "HEALTH_OK\|HEALTH_WARN"; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: Cluster failed to become healthy after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: Cluster not ready, retrying in 5s..." + sleep 5 + done + echo "Cluster is healthy." + + # Wait for RGW availability + echo "Waiting for RGW to be ready..." + ATTEMPT=0 + until curl -sf http://ceph-rgw:7480 >/dev/null 2>&1; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: RGW failed to become ready after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: RGW not ready, retrying in 5s..." + sleep 5 + done + echo "RGW is ready." + + # Idempotent S3 user creation + echo "Provisioning S3 user: ${RGW_ACCESS_KEY}" + if radosgw-admin --cluster ${CLUSTER} user info --uid="${RGW_ACCESS_KEY}" >/dev/null 2>&1; then + echo "User ${RGW_ACCESS_KEY} already exists, skipping creation." + else + echo "Creating new S3 user: ${RGW_ACCESS_KEY}" + radosgw-admin --cluster ${CLUSTER} user create \ + --uid="${RGW_ACCESS_KEY}" \ + --display-name="Object Storage User" \ + --access-key="${RGW_ACCESS_KEY}" \ + --secret-key="${RGW_SECRET_KEY}" + echo "S3 user created successfully." + fi + + echo "Initialization complete. Exiting." + exit 0 + |||, + ]); + + // Container sets - each daemon gets its own for K8s node distribution + local mon_containerSet = engine.containers("ceph-mon", [mon_container]); + local mgr_containerSet = engine.containers("ceph-mgr", [mgr_container]); + local osd_containerSet = engine.containers("ceph-osd", [osd_container]); + local rgw_containerSet = engine.containers("ceph-rgw", [rgw_container]); + local init_containerSet = engine.containers("ceph-init", [init_container]); + + // Services - expose daemon ports for inter-daemon communication + local mon_service = + engine.service(mon_containerSet) + .with_port(6789, 6789, "mon") + .with_port(3300, 3300, "mon-msgr2"); + + local mgr_service = + engine.service(mgr_containerSet) + .with_port(7000, 7000, "mgr") + .with_port(8443, 8443, "dashboard") + .with_port(9283, 9283, "prometheus"); + + local osd_service = + engine.service(osd_containerSet) + .with_port(6800, 6800, "osd"); + + local rgw_service = + engine.service(rgw_containerSet) + .with_port(7480, 7480, "s3"); + + engine.resources([ + + // Data volumes + vol_mon, + vol_mgr, + vol_osd, + vol_rgw, + + // Config volumes (isolated, no sharing) + vol_mon_config, + vol_mgr_config, + vol_osd_config, + vol_rgw_config, + vol_init_config, + + // Container sets + mon_containerSet, + mgr_containerSet, + osd_containerSet, + rgw_containerSet, + init_containerSet, + + // Services + mon_service, + mgr_service, + osd_service, + rgw_service, + + ]) + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/falkordb.jsonnet new file mode 100644 index 00000000..1d4176d8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/falkordb.jsonnet @@ -0,0 +1,38 @@ +local images = import "values/images.jsonnet"; + +{ + + "falkordb" +: { + + create:: function(engine) + + local vol = engine.volume("falkordb").with_size("20G"); + + local container = + engine.container("falkordb") + .with_image(images.falkordb) + .with_limits("1.0", "768M") + .with_reservations("0.5", "768M") + .with_port(6379, 6379, "api") + .with_port(3010, 3000, "ui") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "falkordb", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6379, 6379, "api") + .with_port(3010, 3010, "ui"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/garage.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/garage.jsonnet new file mode 100644 index 00000000..9d339bfb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/garage.jsonnet @@ -0,0 +1,249 @@ +local images = import "values/images.jsonnet"; + +{ + + garage +: { + + // Garage S3 credentials - these are the actual access key ID and secret key + // Access Key ID must be in format: GK + 24 hex characters (12 bytes) + // Secret Key must be 64 hex characters (32 bytes) + // For production, generate secure random values and override these defaults + "access-key":: "GK000000000000000000000001", + "secret-key":: "b171f00be9be4c32c734f4c05fe64c527a8ab5eb823b376cfa8c2531f70fc427", + "rpc-secret":: "bbba746a9e289bad64a9e7a36a4299dac8d6e0b8cc2a6c2937fe756df4492008", + // For a production system, override this value + "admin-token":: "batts-rockhearted-unpartially", + region:: "garage", + "replication-factor":: "1", // Set to 1 for single-node, 3 for production + + // Storage volume sizes + "meta-size":: "2G", // Metadata volume size + "data-size":: "5G", // Data volume size (also used for cluster layout capacity) + + create:: function(engine) + + local accessKey = self["access-key"]; + local secretKey = self["secret-key"]; + local rpcSecret = self["rpc-secret"]; + local adminToken = self["admin-token"]; + local region = self.region; + local replicationFactor = self["replication-factor"]; + local metaSize = self["meta-size"]; + local dataSize = self["data-size"]; + + // Garage daemon configuration file - TOML format + local garage_conf = ||| + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + + db_engine = "lmdb" + + replication_factor = %s + + compression_level = 1 + + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "[::]:3901" + rpc_secret = "%s" + + [s3_api] + s3_region = "%s" + api_bind_addr = "[::]:3900" + root_domain = ".s3.garage.local" + + [s3_web] + bind_addr = "[::]:3902" + root_domain = ".web.garage.local" + index = "index.html" + + [k2v_api] + api_bind_addr = "[::]:3904" + + [admin] + api_bind_addr = "[::]:3903" + admin_token = "%s" + ||| % [replicationFactor, rpcSecret, region, adminToken]; + + // Config volume - contains the rendered garage.toml + local cfgVol = engine.configVolume( + "garage-cfg", "garage", + { + "garage.toml": garage_conf, + } + ); + + // Volumes - Garage stores metadata and data separately + local vol_meta = engine.volume("garage-meta").with_size(metaSize); + local vol_data = engine.volume("garage-data").with_size(dataSize); + + // Main Garage daemon container + local garage_container = + engine.container("garage") + .with_image(images.garage) + .with_command([ + "/garage", "-c", "/etc/garage/garage.toml", "server" + ]) + .with_environment({ + RUST_LOG: "garage=info", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3900, 3900, "s3-api") + .with_port(3901, 3901, "rpc") + .with_port(3902, 3902, "web") + .with_port(3903, 3903, "admin") + .with_port(3904, 3904, "k2v") + .with_volume_mount(cfgVol, "/etc/garage/") + .with_volume_mount(vol_meta, "/var/lib/garage/meta") + .with_volume_mount(vol_data, "/var/lib/garage/data"); + + // Init container - configures cluster layout and creates S3 credentials + // IMPORTANT: This container exits with code 0 after completion + // Orchestrator must NOT restart it (use K8s Job or Compose restart: "no") + // Uses Alpine base image since garage container has no shell + local init_container = + engine.container("garage-init") + .with_image("docker.io/alpine:3.23.2") + .with_environment({ + GARAGE_ACCESS_KEY: accessKey, + GARAGE_SECRET_KEY: secretKey, + GARAGE_REGION: region, + GARAGE_ADMIN_TOKEN: adminToken, + GARAGE_RPC_SECRET: rpcSecret, + GARAGE_DATA_SIZE: dataSize, + }) + .with_limits("0.5", "256M") + .with_reservations("0.25", "128M") + .with_volume_mount(cfgVol, "/etc/garage/") + .with_command([ + "sh", "-c", ||| + set -e + + # Install required tools + echo "Installing curl, jq and downloading garage CLI..." + apk add --no-cache curl jq + + # Download garage binary (v2.1.0) for remote management + curl -fsSL "https://garagehq.deuxfleurs.fr/_releases/v2.1.0/x86_64-unknown-linux-musl/garage" \ + -o /usr/local/bin/garage + chmod +x /usr/local/bin/garage + + echo "Waiting for Garage daemon to be ready..." + MAX_ATTEMPTS=60 + ATTEMPT=0 + # Wait for /health to respond (even 503 is fine - means daemon is up) + until curl -s http://garage:3903/health >/dev/null 2>&1; do + ATTEMPT=$((ATTEMPT+1)) + if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then + echo "ERROR: Garage failed to become ready after ${MAX_ATTEMPTS} attempts" + exit 1 + fi + echo "Attempt ${ATTEMPT}/${MAX_ATTEMPTS}: Garage not ready, retrying in 2s..." + sleep 2 + done + echo "Garage daemon is ready." + + # Get the node ID via v2 Admin API + echo "Getting Garage node ID via Admin API..." + curl -s -H "Authorization: Bearer ${GARAGE_ADMIN_TOKEN}" \ + "http://garage:3903/v2/GetNodeInfo?node=self" > /tmp/garage-node-info.json + + # Extract node ID from response (the key in success map is the node ID) + NODE_ID=$(jq -r '.success | to_entries[0].value.nodeId' /tmp/garage-node-info.json) + echo "Node ID: ${NODE_ID}" + + if [ -z "$NODE_ID" ] || [ "$NODE_ID" = "null" ]; then + echo "ERROR: Failed to retrieve node ID" + exit 1 + fi + + # ===== LAYOUT MANAGEMENT VIA REMOTE RPC ===== + # Use garage CLI with -h and -s flags to connect remotely + # -h requires format: @: + # -s provides the RPC secret + + # Construct full RPC identifier + RPC_HOST="${NODE_ID}@garage:3901" + echo "RPC Host: ${RPC_HOST}" + + # Check current layout to see if node is already assigned (idempotent) + echo "Checking current cluster layout..." + LAYOUT_OUTPUT=$(garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" layout show 2>&1) + + # Check if node already has a role assigned + # Layout output shows abbreviated node ID (first 16 chars) + NODE_ID_SHORT="${NODE_ID:0:16}" + if echo "$LAYOUT_OUTPUT" | grep -q "$NODE_ID_SHORT"; then + echo "Node ${NODE_ID_SHORT}... already assigned in layout, skipping." + else + echo "Assigning node to cluster layout..." + # Assign node to zone dc1 with configured capacity + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + layout assign ${NODE_ID} -z dc1 -c ${GARAGE_DATA_SIZE} + + echo "Applying layout configuration..." + # Get current staged version and apply + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + layout apply --version 1 + + echo "Layout configured successfully." + # Wait for layout to stabilize + sleep 5 + fi + + # ===== KEY MANAGEMENT VIA REMOTE RPC ===== + + # Check if key already exists (idempotent) + # GARAGE_ACCESS_KEY is already a valid Garage Key ID (GK + 24 hex chars) + if garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" key info "${GARAGE_ACCESS_KEY}" >/dev/null 2>&1; then + echo "Access key ${GARAGE_ACCESS_KEY} already exists, skipping creation." + else + echo "Importing S3 access key: ${GARAGE_ACCESS_KEY}" + + # Import key with the provided credentials (both already in valid Garage format) + # GARAGE_ACCESS_KEY = Key ID (GK + 24 hex chars) + # GARAGE_SECRET_KEY = Secret (64 hex chars) + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + key import "${GARAGE_ACCESS_KEY}" "${GARAGE_SECRET_KEY}" --yes + + echo "Access key imported successfully." + fi + + # Grant permissions to the key + echo "Granting create-bucket permission to key..." + garage -h "${RPC_HOST}" -s "${GARAGE_RPC_SECRET}" \ + key allow --create-bucket "${GARAGE_ACCESS_KEY}" + + echo "" + echo "Garage initialization complete!" + echo "S3 Endpoint: http://garage:3900" + echo "Region: ${GARAGE_REGION}" + exit 0 + |||, + ]); + + // Container sets + local garage_containerSet = engine.containers("garage", [garage_container]); + local init_containerSet = engine.containers("garage-init", [init_container]); + + // Service - expose Garage ports + local garage_service = + engine.service(garage_containerSet) + .with_port(3900, 3900, "s3-api") + .with_port(3901, 3901, "rpc") + .with_port(3902, 3902, "web") + .with_port(3903, 3903, "admin") + .with_port(3904, 3904, "k2v"); + + engine.resources([ + cfgVol, + vol_meta, + vol_data, + garage_containerSet, + init_containerSet, + garage_service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/memgraph.jsonnet new file mode 100644 index 00000000..eeed2e4e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/memgraph.jsonnet @@ -0,0 +1,70 @@ +local images = import "values/images.jsonnet"; + +{ + + "memgraph" +: { + + create:: function(engine) + + local vol = engine.volume("memgraph").with_size("20G"); + + local container = + engine.container("memgraph") + .with_image(images.memgraph_mage) + .with_environment({ + MEMGRAPH: "--storage-properties-on-edges=true --storage-enable-edges-metadata=true" + }) + .with_limits("1.0", "1000M") + .with_reservations("0.5", "1000M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/var/lib/memgraph"); + + local containerSet = engine.containers( + "memgraph", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + "memgraph-lab" +: { + + create:: function(engine) + + local container = + engine.container("lab") + .with_image(images.memgraph_lab) + .with_environment({ + QUICK_CONNECT_MG_HOST: "memgraph", + QUICK_CONNECT_MG_PORT: "7687", + }) + .with_limits("1.0", "512M") + .with_reservations("0.5", "512M") + .with_port(3010, 3000, "http"); + + local containerSet = engine.containers( + "lab", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3010, 3010, "http"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/milvus.jsonnet new file mode 100644 index 00000000..5bed3820 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/milvus.jsonnet @@ -0,0 +1,89 @@ +local images = import "values/images.jsonnet"; +local minio = import "backends/minio.jsonnet"; + +minio { + + etcd +: { + + create:: function(engine) + + local vol = engine.volume("etcd").with_size("20G"); + + local container = + engine.container("etcd") + .with_image(images.etcd) + .with_command([ + "etcd", + "-advertise-client-urls=http://127.0.0.1:2379", + "-listen-client-urls", + "http://0.0.0.0:2379", + "--data-dir", + "/etcd", + ]) + .with_environment({ + ETCD_AUTO_COMPACTION_MODE: "revision", + ETCD_AUTO_COMPACTION_RETENTION: "1000", + ETCD_QUOTA_BACKEND_BYTES: "4294967296", + ETCD_SNAPSHOT_COUNT: "50000" + }) + .with_limits("1.0", "128M") + .with_reservations("0.25", "128M") + .with_port(2379, 2379, "api") + .with_volume_mount(vol, "/etcd"); + + local containerSet = engine.containers( + "etcd", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(2379, 2379, "api"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + + milvus +: { + + create:: function(engine) + + local vol = engine.volume("milvus").with_size("20G"); + + local container = + engine.container("milvus") + .with_image(images.milvus) + .with_command([ + "milvus", "run", "standalone" + ]) + .with_environment({ + ETCD_ENDPOINTS: "etcd:2379", + MINIO_ADDRESS: "minio:9000", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2") + .with_volume_mount(vol, "/var/lib/milvus"); + + local containerSet = engine.containers( + "milvus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9091, 9091, "api") + .with_port(19530, 19530, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/minio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/minio.jsonnet new file mode 100644 index 00000000..b38bb81f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/minio.jsonnet @@ -0,0 +1,48 @@ +local images = import "values/images.jsonnet"; + +{ + + minio +: { + + create:: function(engine) + + local vol = engine.volume("minio-data").with_size("20G"); + + local container = + engine.container("minio") + .with_image(images.minio) + .with_command([ + "minio", + "server", + "/minio_data", + "--console-address", + ":9001", + ]) + .with_environment({ + MINIO_ROOT_USER: "minioadmin", + MINIO_ROOT_PASSWORD: "minioadmin", + }) + .with_limits("0.5", "128M") + .with_reservations("0.25", "128M") + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console") + .with_volume_mount(vol, "/minio_data"); + + local containerSet = engine.containers( + "minio", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9000, 9000, "api") + .with_port(9001, 9001, "console"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/neo4j.jsonnet new file mode 100644 index 00000000..46c61e0f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/neo4j.jsonnet @@ -0,0 +1,46 @@ +local images = import "values/images.jsonnet"; + +{ + + "neo4j" +: { + + create:: function(engine) + + local vol = engine.volume("neo4j").with_size("20G"); + + local container = + engine.container("neo4j") + .with_image(images.neo4j) + .with_environment({ + NEO4J_AUTH: "neo4j/password", + NEO4J_server_memory_pagecache_size: "512m", + NEO4J_server_memory_heap_max__size: "512m", + // NEO4J_server_bolt_listen__address: "0.0.0.0:7687", + // NEO4J_server_default__listen__address: "0.0.0.0", + // NEO4J_server_http_listen__address: "0.0.0.0:7474", + }) + .with_limits("1.0", "1536M") + .with_reservations("0.5", "1536M") + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2") + .with_volume_mount(vol, "/data"); + + local containerSet = engine.containers( + "neo4j", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7474, 7474, "api") + .with_port(7687, 7687, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/pinecone.jsonnet new file mode 100644 index 00000000..2bef70fb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/pinecone.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "pinecone" +: { + + create:: function(engine) + + local container = + engine.container("pinecone") + .with_image(images.pinecone) + .with_environment({ + + PORT: "5080", + + INDEX_TYPE: "serverless", + + // Dimension is fixed, 384 is right for miniLM + // sentence embedding + DIMENSION: 384, + + METRIC: "cosine" + + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(5080, 5080, "api"); + + local containerSet = engine.containers( + "pinecone", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(5080, 5080, "api"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/qdrant.jsonnet new file mode 100644 index 00000000..2e6761f8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/backends/qdrant.jsonnet @@ -0,0 +1,65 @@ +local images = import "values/images.jsonnet"; + +{ + + "qdrant" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1024M", + "memory-reservation":: "1024M", + + // Mmap settings for low-memory mode (trades latency for memory) + // Set to null to disable, or string value to enable + "memmap-threshold-kb":: null, + "on-disk-payload":: null, + + create:: function(engine) + + // Capture memory settings into locals + local memLimit = self["memory-limit"]; + local memReserv = self["memory-reservation"]; + local mmapThreshold = self["memmap-threshold-kb"]; + local onDiskPayload = self["on-disk-payload"]; + + local vol = engine.volume("qdrant").with_size("20G"); + + // Build environment with optional mmap settings + local baseEnv = {}; + local env = baseEnv + + (if mmapThreshold != null then { + QDRANT__STORAGE__MEMMAP_THRESHOLD_KB: mmapThreshold, + } else {}) + + (if onDiskPayload != null then { + QDRANT__STORAGE__ON_DISK_PAYLOAD: onDiskPayload, + } else {}); + + local container = + engine.container("qdrant") + .with_image(images.qdrant) + .with_limits("1.0", memLimit) + .with_reservations("0.5", memReserv) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2") + .with_volume_mount(vol, "/qdrant/storage") + + (if std.length(env) > 0 then { + environment+: env, + } else {}); + + local containerSet = engine.containers( + "qdrant", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(6333, 6333, "api") + .with_port(6334, 6334, "api2"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/components.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/components.jsonnet new file mode 100644 index 00000000..132a9361 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/components.jsonnet @@ -0,0 +1,122 @@ +{ + + // Essentials + "trustgraph-base": import "core/trustgraph.jsonnet", + "rev-gateway": import "core/rev-gateway.jsonnet", + "pulsar": import "pubsub/pulsar.jsonnet", + "rabbitmq": import "pubsub/rabbitmq.jsonnet", + + // LLMs + "azure": import "llm/azure.jsonnet", + "azure-openai": import "llm/azure-openai.jsonnet", + "bedrock": import "llm/bedrock.jsonnet", + "claude": import "llm/claude.jsonnet", + "cohere": import "llm/cohere.jsonnet", + "googleaistudio": import "llm/googleaistudio.jsonnet", + "llamafile": import "llm/llamafile.jsonnet", + "lmstudio": import "llm/lmstudio.jsonnet", + "mistral": import "llm/mistral.jsonnet", + "ollama": import "llm/ollama.jsonnet", + "openai": import "llm/openai.jsonnet", + "vertexai": import "llm/vertexai.jsonnet", + "tgi": import "llm/tgi.jsonnet", + "vllm": import "llm/vllm.jsonnet", + + // Embeddings + "embeddings-ollama": import "embeddings/embeddings-ollama.jsonnet", + "embeddings-hf": import "embeddings/embeddings-hf.jsonnet", + "embeddings-fastembed": import "embeddings/embeddings-fastembed.jsonnet", + + // OCR options + "ocr": import "ocr/ocr.jsonnet", + "mistral-ocr": import "ocr/mistral-ocr.jsonnet", + + // Vector stores + "vector-store-milvus": import "vector-store/milvus.jsonnet", + "vector-store-qdrant": import "vector-store/qdrant.jsonnet", + "vector-store-pinecone": import "vector-store/pinecone.jsonnet", + + // Triples stores + "triple-store-cassandra": import "triple-store/cassandra.jsonnet", + "triple-store-neo4j": import "triple-store/neo4j.jsonnet", + "triple-store-falkordb": import "triple-store/falkordb.jsonnet", + "triple-store-memgraph": import "triple-store/memgraph.jsonnet", + + // Row stores + "row-store-cassandra": import "row-store/cassandra.jsonnet", + + // Observability support + "grafana": import "monitoring/grafana.jsonnet", + "loki": import "monitoring/loki.jsonnet", + + // Pulsar manager is a UI for Pulsar. Uses a LOT of memory + "pulsar-manager": import "pulsar/pulsar-manager.jsonnet", + + "override-recursive-chunker": import "core/chunker-recursive.jsonnet", + + // The prompt manager + "prompt-overrides": import "core/prompt-overrides.jsonnet", + + // Extra MCP services + "ddg-mcp-server": import "mcp/ddg-mcp-server.jsonnet", + + // Does nothing. But, can be a hack to overwrite parameters + "null": {}, + + // Passthrough: returns parameters directly, preserving +: merge syntax + // Also supports JSON-safe routing with prefixed parameters like "cassandra-heap" + "override": { + local route = function(target) + function(prefix, k, v) + local suffix = std.substr(k, std.length(prefix), std.length(k) - std.length(prefix)); + { [target] +: { [suffix]:: v } }, + + local routes = { + "cassandra-": route("cassandra"), + "pulsar-": route("pulsar"), + "qdrant-": route("qdrant"), + "api-gateway-": route("api-gateway"), + "librarian-": route("librarian"), + }, + + with_params:: function(pars) + std.foldl( + function(acc, k) + local matchingPrefixes = [p for p in std.objectFields(routes) if std.startsWith(k, p)]; + if std.length(matchingPrefixes) > 0 then + local prefix = matchingPrefixes[0]; + acc + routes[prefix](prefix, k, pars[k]) + else + acc + { [k]:: pars[k] }, + std.objectFields(pars), + {} + ), + }, + + // Memory profiles + "memory-profile-low": import "profiles/memory-profile-low.jsonnet", + + // Model hosting + + "hosting-intel-battlemage-vllm": + import "model-hosting/intel-battlemage-vllm.jsonnet", + + "hosting-cpu-tgi": + import "model-hosting/cpu-tgi.jsonnet", + + "hosting-intel-xpu-tgi": + import "model-hosting/intel-xpu-tgi.jsonnet", + + "hosting-intel-gaudi-tgi": + import "model-hosting/intel-gaudi-tgi.jsonnet", + + "hosting-intel-xpu-vllm": + import "model-hosting/intel-xpu-vllm.jsonnet", + + "hosting-intel-gaudi-vllm": + import "model-hosting/intel-gaudi-vllm.jsonnet", + + "hosting-nvidia-gpu-vllm": + import "model-hosting/nvidia-gpu-vllm.jsonnet", + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/agent-manager-react.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/agent-manager-react.jsonnet new file mode 100644 index 00000000..2d3d3340 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/agent-manager-react.jsonnet @@ -0,0 +1,46 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "agent-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("agent-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "agent-manager-react", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "agent-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/agent-orchestrator.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/agent-orchestrator.jsonnet new file mode 100644 index 00000000..60cfaed9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/agent-orchestrator.jsonnet @@ -0,0 +1,46 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "agent-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("agent-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "agent-orchestrator", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "agent-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/chunker-recursive.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/chunker-recursive.jsonnet new file mode 100644 index 00000000..cfdd571e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/chunker-recursive.jsonnet @@ -0,0 +1,54 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "chunk-size":: 2000, + "chunk-overlap":: 100, + + "chunker" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + ] + $["pub-sub-args"] + [ + "--chunk-size", + std.toString($["chunk-size"]), + "--chunk-overlap", + std.toString($["chunk-overlap"]), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/configuration.jsonnet new file mode 100644 index 00000000..e5b979ce --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/configuration.jsonnet @@ -0,0 +1,56 @@ + +// This puts the default configuration together. References many things, +// flow classes, a default flow, token costs, prompts, agent tools + +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "init-trustgraph" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local cfgVol = engine.configVolume( + "trustgraph-cfg", "trustgraph", + { + "config.json": importstr "trustgraph/config.json", + } + ); + + local container = + engine.container("init-trustgraph") + .with_image(images.trustgraph_flow) + .with_command( + [ + "tg-init-trustgraph", + ] + $["pub-sub-args"] + $["pub-sub-init-args"] + [ + "--config-file", + "/trustgraph/config.json", + ] + ) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_volume_mount(cfgVol, "/trustgraph/"); + + local containerSet = engine.containers( + "init-trustgraph", [ container ] + ); + + engine.resources([ + cfgVol, + containerSet, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/document-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/document-rag.jsonnet new file mode 100644 index 00000000..acb2095a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/document-rag.jsonnet @@ -0,0 +1,90 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + "document-rag" +: { + + "doc-limit":: 20, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local docLimit = self["doc-limit"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "document-rag", + ] + $["pub-sub-args"] + [ + "--doc-limit", + std.toString(docLimit), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "document-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "document-embeddings", +] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/graph-rag.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/graph-rag.jsonnet new file mode 100644 index 00000000..5c4385c9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/graph-rag.jsonnet @@ -0,0 +1,285 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "kg-extract-definitions" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-definitions") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-definitions", +] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-definitions", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-relationships" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-relationships") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-relationships", +] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-relationships", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-agent" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-agent") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-agent", +] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-agent", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-ontology" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "300M", + "memory-reservation":: "300M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-ontology") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-ontology", +] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-ontology", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-rag" +: { + + concurrency:: 1, + "entity-limit":: 50, + "triple-limit":: 30, + "edge-limit":: 30, + "edge-score-limit":: 10, + "max-subgraph-size":: 100, + "max-path-length":: 2, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local entityLimit = self["entity-limit"]; + local tripleLimit = self["triple-limit"]; + local edgeLimit = self["edge-limit"]; + local edgeScoreLimit = self["edge-score-limit"]; + local maxSubgraphSize = self["max-subgraph-size"]; + local maxPathLength = self["max-path-length"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("graph-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-rag", + ] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "--entity-limit", + std.toString(entityLimit), + "--triple-limit", + std.toString(tripleLimit), + "--edge-limit", + std.toString(edgeLimit), + "--edge-score-limit", + std.toString(edgeScoreLimit), + "--max-subgraph-size", + std.toString(maxSubgraphSize), + "--max-path-length", + std.toString(maxPathLength), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "graph-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "graph-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/librarian.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/librarian.jsonnet new file mode 100644 index 00000000..6278edcf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/librarian.jsonnet @@ -0,0 +1,57 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local garage = import "backends/garage.jsonnet"; +local cassandra = import "backends/cassandra.jsonnet"; + +{ + + "librarian" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("librarian") + .with_image(images.trustgraph_flow) + .with_command([ + "librarian", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + "--object-store-endpoint", + url.object_store, + "--object-store-access-key", + $.garage["access-key"], + "--object-store-secret-key", + $.garage["secret-key"], + "--object-store-region", + $.garage.region, + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "librarian", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +// Garage and Cassandra are used by the Librarian +} + garage + cassandra + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/mcp-server.jsonnet new file mode 100644 index 00000000..9bf7e3bd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/mcp-server.jsonnet @@ -0,0 +1,54 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "mcp-server" +: { + + port:: 8000, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local port = self.port; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("mcp-server-secret") + .with_env_var("MCP_SERVER_SECRET", "mcp-server-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local container = + engine.container("mcp-server") + .with_image(images.trustgraph_mcp) + .with_command([ + "mcp-server", + "--port", + std.toString(port), + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/prompt-overrides.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/prompt-overrides.jsonnet new file mode 100644 index 00000000..852ec09d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/prompt-overrides.jsonnet @@ -0,0 +1,24 @@ +local default_prompts = import "prompts/default-prompts.jsonnet"; + +{ + + with:: function(key, value) + if (key == "system-template") then + self + { + prompts +:: { + "system-template": value, + } + } + else + self + { + prompts +:: { + templates +:: { + [key] +:: { + prompt: value + } + } + } + }, + +} + default_prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/prompt-template.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/prompt-template.jsonnet new file mode 100644 index 00000000..23ac1442 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/prompt-template.jsonnet @@ -0,0 +1,95 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "prompt" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("prompt") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + ] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "prompt", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "prompt-rag" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("prompt-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "prompt-template", + ] + $["pub-sub-args"] + [ + "--id", + "prompt-rag", + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "prompt-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/rev-gateway.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/rev-gateway.jsonnet new file mode 100644 index 00000000..39b71e31 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/rev-gateway.jsonnet @@ -0,0 +1,61 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "rev-gateway" +: { + + // Invalid, but at least means the rev-gateway won't connect to anything + // it shouldn't. + token:: "INVALID_TOKEN", + uri:: "wss://127.0.0.1/api/v1/relay?token=" + self.token, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local uri = self.uri; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("rev-gateway-secret") + .with_env_var("REV_GATEWAY_SECRET", "rev-gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "rev-gateway", + ] + $["pub-sub-args"] + [ + "--websocket-uri", + std.toString(uri), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/structured-data.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/structured-data.jsonnet new file mode 100644 index 00000000..06956b27 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/structured-data.jsonnet @@ -0,0 +1,206 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "nlp-query" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("nlp-query") + .with_image(images.trustgraph_flow) + .with_command([ + "nlp-query", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "nlp-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-query" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("structured-query") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-query", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "structured-query", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "structured-diag" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "96M", + "memory-reservation":: "96M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("structured-diag") + .with_image(images.trustgraph_flow) + .with_command([ + "structured-diag", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "structured-diag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-extract-rows" +: { + + concurrency:: 1, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local concurrency = self.concurrency; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-extract-rows") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-extract-rows", + ] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-extract-rows", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "row-embeddings" +: { + + "cpu-limit":: "1.0", + "cpu-reservation":: "0.5", + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("row-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "row-embeddings", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "row-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/trustgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/trustgraph.jsonnet new file mode 100644 index 00000000..571a3453 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/core/trustgraph.jsonnet @@ -0,0 +1,464 @@ +local images = import "values/images.jsonnet"; + +local config_initialiser = import "configuration.jsonnet"; +local config = import "../runtime-config/trustgraph-config.jsonnet"; +local librarian = import "librarian.jsonnet"; +local mcp_server = import "mcp-server.jsonnet"; +local workbench = import "../ui/workbench-ui.jsonnet"; +local graphrag = import "graph-rag.jsonnet"; +local documentrag = import "document-rag.jsonnet"; +local prompt_template = import "prompt-template.jsonnet"; +local agent_manager = import "agent-orchestrator.jsonnet"; +local structured_data = import "structured-data.jsonnet"; +local ddg = import "mcp/ddg-mcp-server.jsonnet"; + +// Helper to create a routing function for a target object +local route = function(target) + function(prefix, k, v) + local suffix = std.substr(k, std.length(prefix), std.length(k) - std.length(prefix)); + { [target] +: { [suffix]:: v } }; + +// Parameter prefix -> target object routing table +local routes = { + "prompt-rag-": route("prompt-rag"), + "prompt-": route("prompt"), + "text-completion-rag-": route("text-completion-rag"), + "text-completion-": route("text-completion"), + "embeddings-": route("embeddings"), + "api-gateway-": route("api-gateway"), + "chunk-": route("chunker"), + "graph-rag-": route("graph-rag"), + "graph-embeddings-": route("graph-embeddings"), + "kg-extract-definitions-": route("kg-extract-definitions"), + "kg-extract-relationships-": route("kg-extract-relationships"), + "kg-extract-agent-": route("kg-extract-agent"), + "kg-extract-ontology-": route("kg-extract-ontology"), + "kg-extract-objects-": route("kg-extract-objects"), + "garage-": route("garage"), + "config-svc-": route("config-svc"), + "document-decoder-": route("document-decoder"), + "mcp-tool-": route("mcp-tool"), + "mcp-server-": route("mcp-server"), + "metering-rag-": route("metering-rag"), + "metering-": route("metering"), + "kg-store-": route("kg-store"), + "kg-manager-": route("kg-manager"), + "librarian-": route("librarian"), + "agent-manager-": route("agent-manager"), + "document-rag-": route("document-rag"), + "document-embeddings-": route("document-embeddings"), + "rev-gateway-": route("rev-gateway"), + "nlp-query-": route("nlp-query"), + "structured-query-": route("structured-query"), + "structured-diag-": route("structured-diag"), + "init-trustgraph-": route("init-trustgraph"), +}; + +// Find longest matching prefix (most specific first) +local findRoute = function(k) + local prefixes = std.objectFields(routes); + local matching = std.filter(function(p) std.startsWith(k, p), prefixes); + local sorted = std.sort(matching, function(x) -std.length(x)); + if std.length(sorted) > 0 then sorted[0] else null; + +{ + + // Route parameters to appropriate internal objects based on prefix + with:: function(k, v) + local prefix = findRoute(k); + if prefix != null then + self + routes[prefix](prefix, k, v) + else + self + { [k]:: v }, + + "log-level":: "INFO", + + // Base objects with concurrency defaults (LLM/embeddings components + // merge into these) + "text-completion" +: { concurrency:: 1 }, + "text-completion-rag" +: { concurrency:: 1 }, + embeddings +: { concurrency:: 1 }, + + "api-gateway" +: { + + port:: 8088, + timeout:: 600, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local port = self.port; + local timeout = self.timeout; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local envSecrets = engine.envSecrets("gateway-secret") + .with_env_var("GATEWAY_SECRET", "gateway-secret"); + + local container = + engine.container("api-gateway") + .with_image(images.trustgraph_flow) + .with_command([ + "api-gateway", + ] + $["pub-sub-args"] + [ + "--timeout", + std.toString(timeout), + "--port", + std.toString(port), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_port(port, port, "api"); + + local containerSet = engine.containers( + "api-gateway", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics") + .with_port(port, port, "api"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "chunker" +: { + + size:: 2000, + overlap:: 50, + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local size = self.size; + local overlap = self.overlap; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("chunker") + .with_image(images.trustgraph_flow) + .with_command([ + "chunker-recursive", + ] + $["pub-sub-args"] + [ + "--chunk-size", + std.toString(size), + "--chunk-overlap", + std.toString(overlap), + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "chunker", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "config-svc" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local cpuLimit = self["cpu-limit"]; + local cpuReservation = self["cpu-reservation"]; + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("config-svc") + .with_image(images.trustgraph_flow) + .with_command([ + "config-svc", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(cpuLimit, memoryLimit) + .with_reservations(cpuReservation, memoryReservation); + + local containerSet = engine.containers( + "config-svc", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "document-decoder" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "512M", + "memory-reservation":: "512M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("document-decoder") + .with_image(images.trustgraph_unstructured) + .with_command([ + "universal-decoder", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "document-decoder", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "mcp-tool" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("mcp-tool") + .with_image(images.trustgraph_flow) + .with_command([ + "mcp-tool", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "mcp-tool", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("metering") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "metering", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "metering-rag" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("metering-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "metering", + ] + $["pub-sub-args"] + [ + "--id", + "metering-rag", + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "metering-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-store" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-store") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-store", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-store", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "kg-manager" +: { + + "cpu-limit":: "0.5", + "cpu-reservation":: "0.1", + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("kg-manager") + .with_image(images.trustgraph_flow) + .with_command([ + "kg-manager", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation); + + local containerSet = engine.containers( + "kg-manager", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + librarian + mcp_server + workbench + graphrag + + documentrag + prompt_template + agent_manager + structured_data + + config_initialiser + config + + ddg + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/embeddings/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/embeddings/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..17b4b0d5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/embeddings/embeddings-fastembed.jsonnet @@ -0,0 +1,48 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-fastembed.jsonnet"; + +{ + + "fastembed-models":: models, + + "embeddings-models" +:: $["fastembed-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-fastembed", + ] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_limits("2.0", "600M") + .with_reservations("1.0", "600M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/embeddings/embeddings-hf.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/embeddings/embeddings-hf.jsonnet new file mode 100644 index 00000000..54c63fb8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/embeddings/embeddings-hf.jsonnet @@ -0,0 +1,46 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/embeddings-huggingface.jsonnet"; + +{ + + "huggingface-embeddings-models":: models, + + "embeddings-models" +:: $["huggingface-embeddings-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_hf) + .with_command([ + "embeddings-hf", + ] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + ]) + .with_limits("1.0", "400M") + .with_reservations("0.5", "400M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/embeddings/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/embeddings/embeddings-ollama.jsonnet new file mode 100644 index 00000000..2d1772d3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/embeddings/embeddings-ollama.jsonnet @@ -0,0 +1,51 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local models = import "parameters/embeddings-ollama.jsonnet"; + +{ + + "ollama-url":: "${OLLAMA_HOST}", + + "ollama-models":: models, + + "embeddings-models" +:: $["ollama-models"], + + embeddings +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local container = + engine.container("embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "embeddings-ollama", + ] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "-r", + $["ollama-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/aks-k8s.jsonnet new file mode 100644 index 00000000..07e050f2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/aks-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "disk.csi.azure.com", + parameters: { + // Standard disks (spinning magnetic), Locally Redundant Storage + // Cheapest, basically + skuName: "Standard_LRS", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/docker-compose.jsonnet new file mode 100644 index 00000000..334b3517 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/docker-compose.jsonnet @@ -0,0 +1,260 @@ +{ + + // Extract resources using the engine + package:: function(patterns) + std.foldl( + function(state, p) state + p.create(self), + std.objectValues(patterns), + {} + ), + + container:: function(name) + { + + local container = self, + + name:: name, + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_group:: function(x) self + + if std.objectHas(container, "group_add") then + { group_add: container.group_add + [x] } + else + { group_add: [x] }, + + with_command:: function(x) self + { + command: + if std.isString(x) then + std.strReplace(x, "$", "$$") + else if std.isArray(x) then + std.map(function(s) std.strReplace(s, "$", "$$"), x) + else + x + }, + + with_entrypoint:: function(x) self + { entrypoint: x }, + + with_runtime:: function(x) self + { runtime: x }, + + with_privileged:: function(x) self + { privileged: x }, + + with_ipc:: function(x) self + { ipc: x }, + + with_capability:: function(x) self + + if std.objectHas(container, "capability") then + { cap_add: container.capability + x } + else + { cap_add: [x], }, + + with_environment:: function(x) self + + if std.objectHas(container, "environment") then + { environment: container.environment + x } + else + { environment: x, }, + + with_device:: function(hdev, cdev) self + + if std.objectHas(container, "devices") then + { devices: container.devices + [ "%s:%s" % [hdev, cdev] ] } + else + { devices: [ "%s:%s" % [hdev, cdev] ], }, + + with_limits:: function(c, m) self + { + deploy +: { resources +: { + limits: { cpus: c, memory: m } + } }, + }, + + with_reservations:: function(c, m) self + { + deploy +: { resources +: { + reservations: { cpus: c, memory: m } + } }, + }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [vol.volid, mnt] + ] + else + [ + "%s:%s" % [vol.volid, mnt] + ] + }, + + with_bind_mount:: + function(src, dest) + self + { + volumes: + if std.objectHas(container, "volumes") then + container.volumes + [ + "%s:%s" % [src, dest] + ] + else + [ + "%s:%s" % [src, dest] + ] + }, + + with_port:: + function(src, dest, name) + self + { + ports: + if std.objectHas(container, "ports") then + container.ports + [ "%d:%d" % [src, dest] ] + else + [ "%d:%d" % [src, dest] ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj.with_environment( + { [x]: "${" + x + "}" } + ), + vars.variables, + self + ), + + restart: "on-failure:100", + + add:: function() { + services +: { + [container.name]: container, + } + } + + }, + + internalService:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + with_port:: function(src, dest, name) + self + { port: [src, dest] }, + + add:: function() { + } + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + with_size:: function(size) self + { size: size }, + + add:: function() { + volumes +: { + [volume.name]: {} + } + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: "./" + dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: dir, + + volid:: dir, + + with_size:: function(size) self + { size: size }, + + add:: function() { + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + volid:: name, + + variables:: [], + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + }, + + add:: function() { + } + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.foldl( + function(state, c) state + c.add(), + cont.containers, + {} + ), + + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/eks-k8s.jsonnet new file mode 100644 index 00000000..3fc3f035 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/eks-k8s.jsonnet @@ -0,0 +1,46 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "ebs.csi.aws.com", + parameters: { + type: "gp3", + encrypted: "true", + iops: "6000", + throughput: "400", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/gcp-k8s.jsonnet new file mode 100644 index 00000000..71792426 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/gcp-k8s.jsonnet @@ -0,0 +1,44 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "pd.csi.storage.gke.io", + parameters: { + type: "pd-balanced", + "csi.storage.k8s.io/fstype": "ext4", + }, + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/k8s.jsonnet new file mode 100644 index 00000000..2067f6ac --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/k8s.jsonnet @@ -0,0 +1,393 @@ +{ + + container:: function(name) + { + + local container = self, + + name: name, + limits: {}, + reservations: {}, + ports: [], + volumes: [], + bindMounts: [], + groups: [], + environment: [], + + with_image:: function(x) self + { image: x }, + + with_user:: function(x) self + { user: x }, + + with_group:: function(x) self + { groups: super.groups + [x] }, + + with_privileged:: function(x) self + { privileged: x }, + + with_command:: function(x) self + { command: x }, + + with_entrypoint:: function(x) self + { entrypoint: x }, + + with_environment:: function(x) self + { + environment: super.environment + [ + { + name: v.key, value: v.value + } + for v in std.objectKeysValues(x) + ], + }, + + with_limits:: function(c, m) self + { limits: { cpu: c, memory: m } }, + + with_reservations:: + function(c, m) self + { reservations: { cpu: c, memory: m } }, + + with_volume_mount:: + function(vol, mnt) + self + { + volumes: super.volumes + [{ + volume: vol, mount: mnt + }] + }, + + with_bind_mount:: + function(src, dest) + local name = "bind-" + std.strReplace(std.strReplace(src, "/", "-"), ".", "-"); + self + { + bindMounts: super.bindMounts + [{ + name: name, src: src, dest: dest + }] + }, + + with_port:: + function(src, dest, name) self + { + ports: super.ports + [ + { src: src, dest: dest, name : name } + ] + }, + + with_env_var_secrets:: + function(vars) + std.foldl( + function(obj, x) obj + { + environment: super.environment + [{ + name: x, + valueFrom: { + secretKeyRef: { + name: vars.name, + key: vars.keyMap[x], + } + } + }] + }, + vars.variables, + self + ), + + add:: function() [ + + { + apiVersion: "apps/v1", + kind: "Deployment", + metadata: { + name: container.name, + namespace: "trustgraph", + labels: { + app: container.name + } + }, + spec: { + replicas: 1, + selector: { + matchLabels: { + app: container.name, + } + }, + template: { + metadata: { + labels: { + app: container.name, + } + }, + spec: { + containers: [ + { + name: container.name, + image: container.image, + + // FIXME: Make everything run as + // root. Needed to get filesystems + // to be accessible. There's a + // better way of doing this? + securityContext: { + runAsUser: 0, + runAsGroup: 0, + } + ( + if std.objectHas(container, "privileged") && container.privileged then + { privileged: true } + else {} + ), + + resources: { + requests: container.reservations, + limits: container.limits + }, + } + ( + if std.length(container.ports) > 0 then + { + ports: [ + { + hostPort: port.src, + containerPort: port.dest, + } + for port in container.ports + ] + } else + {}) + + + (if std.objectHas(container, "entrypoint") then + // Entrypoint is set - use command for entrypoint, args for command + (if std.isString(container.entrypoint) && container.entrypoint == "" then + { command: [] } + else if std.isArray(container.entrypoint) then + { command: container.entrypoint } + else + { command: [container.entrypoint] } + ) + (if std.objectHas(container, "command") then + { args: container.command } + else {}) + else if std.objectHas(container, "command") then + { command: container.command } + else {}) + + + (if std.length(container.environment) > 0 then + { + env: container.environment, + } + else {}) + + + (if std.length(container.volumes) > 0 || std.length(container.bindMounts) > 0 then + { + volumeMounts: [ + { + mountPath: vol.mount, + name: vol.volume.name, + } + for vol in container.volumes + ] + [ + { + mountPath: bm.dest, + name: bm.name, + } + for bm in container.bindMounts + ] + } + + else + {} + ) + ], + volumes: [ + vol.volume.volRef() + for vol in container.volumes + ] + [ + { + name: bm.name, + hostPath: { path: bm.src } + } + for bm in container.bindMounts + ] + } + ( + if std.length(container.groups) > 0 then + { securityContext: { supplementalGroups: container.groups } } + else {} + ) + }, + } + {} + + } + + ] + + }, + + // Just an alias + internalService:: self.service, + + service:: function(containers) + { + + local service = self, + + name: containers.name, + + ports: [], + + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + + add:: function() [ + + { + + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + + }, + + volume:: function(name) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + storageClassName: "tg", + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + configVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + { + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + data: parts + }, + ], + + + volRef:: function() { + name: volume.name, + configMap: { name: volume.name }, + } + + }, + + secretVolume:: function(name, dir, parts) + { + + local volume = self, + + name: name, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + } + + }, + + envSecrets:: function(name) + { + + local volume = self, + + name: name, + + variables: [], + keyMap: {}, + + with_size:: function(size) self + { size: size }, + + add:: function() [ + ], + + volRef:: function() { + name: volume.name, + secret: { secretName: volume.name }, + }, + + with_env_var:: + function(name, key) self + { + variables: super.variables + [name], + keyMap: super.keyMap + { [name]: key }, + }, + + }, + + containers:: function(name, containers) + { + + local cont = self, + + name: name, + containers: containers, + + add:: function() std.flattenArrays( + [ c.add() for c in cont.containers ] + ), + + }, + + resources:: function(res) + + std.flattenArrays( + [ c.add() for c in res ] + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/minikube-k8s.jsonnet new file mode 100644 index 00000000..858b17ad --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/minikube-k8s.jsonnet @@ -0,0 +1,115 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: resources, + }; + resourceList, + + volume:: function(name) + { + local volume = self, + name: name, + with_size:: function(size) self + { size: size }, + add:: function() [ + { + apiVersion: "v1", + kind: "PersistentVolume", + metadata: { + name: volume.name, + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + capacity: { + storage: volume.size, + }, + persistentVolumeReclaimPolicy: "Delete", + hostPath: { + path: "/data/pv-" + volume.name, + }, + } + }, + { + apiVersion: "v1", + kind: "PersistentVolumeClaim", + metadata: { + name: volume.name, + namespace: "trustgraph", + }, + spec: { + accessModes: [ "ReadWriteOnce" ], + resources: { + requests: { + storage: volume.size, + } + }, + } + } + ], + + volRef:: function() { + name: volume.name, + persistentVolumeClaim: { claimName: volume.name }, + } + + }, + + service:: function(containers) + { + local service = self, + name: containers.name, + ports: [], + with_port:: + function(src, dest, name) + self + { + ports: super.ports + [ + { src: src, dest: dest, name: name } + ] + }, + add:: function() [ + { + apiVersion: "v1", + kind: "Service", + metadata: { + name: service.name, + namespace: "trustgraph", + }, + spec: { + selector: { + app: service.name, + }, + type: "LoadBalancer", + ports: [ + { + port: port.src, + targetPort: port.dest, + name: port.name, + } + for port in service.ports + ], + } + } + ], + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/noop.jsonnet new file mode 100644 index 00000000..1f384648 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/noop.jsonnet @@ -0,0 +1,79 @@ +{ + + // Extract resources usnig the engine + package:: function(patterns) {}, + + container:: function(name) { + + with_image:: function(x) self + {}, + + with_user:: function(x) self + {}, + + with_command:: function(x) self + {}, + + with_runtime:: function(x) self + {}, + + with_privileged:: function(x) self + {}, + + with_ipc:: function(x) self + {}, + + with_capability:: function(x) self + {}, + + with_environment:: function(x) self + {}, + + with_device:: function(hdev, cdev) self + {}, + + with_limits:: function(c, m) self + {}, + + with_reservations:: function(c, m) self + {}, + + with_volume_mount:: self + {}, + + with_port:: function(src, dest, name) self + {}, + + with_env_var_secrets:: function(vars) self + {}, + + add:: function() {}, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + service:: function(containers) { + with_port:: function(src, dest, name) self + {}, + add:: function() {}, + }, + + volume:: function(name) { + with_size:: function(size) self + {}, + add:: function() {}, + }, + + configVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + secretVolume:: function(name, dir, parts) { + add:: function() {}, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self + {}, + add:: function() {}, + }, + + containers:: function(name, containers) { + add:: function() {}, + }, + + resources:: function(res) + std.foldl( + function(state, c) state + c.add(), + res, + {} + ), + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/ovh-k8s.jsonnet new file mode 100644 index 00000000..15d57c83 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/ovh-k8s.jsonnet @@ -0,0 +1,45 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "cinder.csi.openstack.org", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", + parameters: { + availability: "nova", + fsType: "ext4", + type: "high-speed", + }, +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/scw-k8s.jsonnet new file mode 100644 index 00000000..eed23968 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/engine/scw-k8s.jsonnet @@ -0,0 +1,40 @@ + +local k8s = import "k8s.jsonnet"; + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +local sc = { + apiVersion: "storage.k8s.io/v1", + kind: "StorageClass", + metadata: { + name: "tg", + }, + provisioner: "csi.scaleway.com", + reclaimPolicy: "Delete", + volumeBindingMode: "WaitForFirstConsumer", +}; + +k8s + { + + // Extract resources usnig the engine + package:: function(patterns) + local resources = [sc, ns] + std.flattenArrays([ + p.create(self) for p in std.objectValues(patterns) + ]); + local resourceList = { + apiVersion: "v1", + kind: "List", + items: [ns, sc] + resources, + }; + resourceList + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/agent-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/agent-extract.jsonnet new file mode 100644 index 00000000..4c5785c8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/agent-extract.jsonnet @@ -0,0 +1,35 @@ +// Agent-based extraction module +// Uses AI agents for more sophisticated knowledge extraction from text +// Leverages agent tools and reasoning for complex extraction tasks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - internal agent extraction service + "interfaces" +: { + }, + + // No configurable parameters for agent extraction + "parameters" +: { + }, + + // Flow-level processors for agent-based extraction + "flow" +: { + // Agent-based knowledge extraction processor + // Uses AI agents with tools to extract structured knowledge + "kg-extract-agent:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output knowledge triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "agent-request": request("agent:{id}"), // Agent service requests + "agent-response": response("agent:{id}"), // Agent service responses + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/agent.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/agent.jsonnet new file mode 100644 index 00000000..7584cdf5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/agent.jsonnet @@ -0,0 +1,60 @@ +// Agent management module +// Provides AI agent orchestration and tool integration +// Manages agent conversations, tool calls, and response coordination +// Supports MCP tools, GraphRAG, and structured queries + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services (agent requires LLM for reasoning, MCP for tools) +local llm_services = import "llm-services.jsonnet"; +local mcp_service = import "mcp-service.jsonnet"; + +// Merge shared services with agent-specific configuration +llm_services + mcp_service + { + + // External interfaces for agent operations + "interfaces" +: { + "agent": request_response("agent:{id}"), + }, + + // Flow-level processors for agent management + "flow" +: { + // Agent manager orchestrates agent conversations and tool usage + "agent-manager:{id}": { + + // Agent communication channels + request: request("agent:{id}"), + next: request("agent:{id}"), + response: response("agent:{id}"), + + // LLM and prompt services + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + + // Tool integrations + "mcp-tool-request": request("mcp-tool:{id}"), + "mcp-tool-response": response("mcp-tool:{id}"), + "graph-rag-request": request("graph-rag:{id}"), + "graph-rag-response": response("graph-rag:{id}"), + "structured-query-request": request("structured-query:{id}"), + "structured-query-response": response("structured-query:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "row-embeddings-query-request": request("row-embeddings:{id}"), + "row-embeddings-query-response": response("row-embeddings:{id}"), + + // Explainability + explainability: flow("triples-store:{id}"), + }, + }, + + // Blueprint-level processors for agent-related services + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/document-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/document-store.jsonnet new file mode 100644 index 00000000..769718fa --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/document-store.jsonnet @@ -0,0 +1,57 @@ +// Document store module +// Infrastructure for document-based RAG using chunk embeddings +// Handles document embedding storage, retrieval, and question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with document store configuration +llm_services + embeddings_service + { + + // External interfaces for document store + "interfaces" +: { + // Document embedding storage and retrieval + "document-embeddings-store": flow("document-embeddings-store:{id}"), + "document-rag": request_response("document-rag:{id}"), + "document-embeddings": request_response("document-embeddings:{id}"), + }, + + // Flow-level processors for document embedding and storage + "flow" +: { + "document-embeddings:{id}": { + input: flow("chunk-load:{id}"), + output: flow("document-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "doc-embeddings-write:{id}": { + input: flow("document-embeddings-store:{id}"), + }, + "document-rag:{id}": { + request: request("document-rag:{id}"), + response: response("document-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "document-embeddings-request": request("document-embeddings:{id}"), + "document-embeddings-response": response("document-embeddings:{id}"), + explainability: flow("triples-store:{id}"), + }, + "doc-embeddings-query:{id}": { + request: request("document-embeddings:{id}"), + response: response("document-embeddings:{id}"), + }, + }, + + // Blueprint-level processors for document RAG operations + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/embeddings-service.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/embeddings-service.jsonnet new file mode 100644 index 00000000..1537e6c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/embeddings-service.jsonnet @@ -0,0 +1,30 @@ +// Shared embeddings service module +// Provides vector embedding generation for text +// Import this module in any flow that requires embeddings + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // Interfaces exposed by embeddings service + "interfaces" +: { + "embeddings": request_response("embeddings:{id}"), + }, + + "parameters" +: { + }, + + // Flow-level processor for embeddings + "flow" +: { + "embeddings:{id}": { + request: request("embeddings:{id}"), + response: response("embeddings:{id}"), + model: "{embeddings-model}", + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/flow-blueprints.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/flow-blueprints.jsonnet new file mode 100644 index 00000000..8f054690 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/flow-blueprints.jsonnet @@ -0,0 +1,48 @@ +// TrustGraph Flow Blueprints Configuration +// +// RAG Modes (4 types): +// - Document RAG: Uses document chunk embeddings +// - Graph RAG: Extracts definitions + relationships to graph +// - Ontology RAG: Extracts using ontology definitions to graph (mutually exclusive with Graph RAG) +// - Structured RAG: Extracts objects to object store +// +// Module structure: +// - *-store: Storage and query infrastructure +// - *-extract: Extraction methods + +// Import all the modular flow components +local graph_store = import "graph-store.jsonnet"; +local document_store = import "document-store.jsonnet"; +local structured_store = import "structured-store.jsonnet"; +local graphrag_extract = import "graphrag-extract.jsonnet"; +local ontorag_extract = import "ontorag-extract.jsonnet"; +local structured_extract = import "structured-extract.jsonnet"; +local agent = import "agent.jsonnet"; +local load = import "load.jsonnet"; +local kgcore = import "kgcore.jsonnet"; + +{ + + // Full system: Graph RAG + Document RAG + knowledge cores + "everything": { + description: "Graph RAG + Document RAG + knowledge cores", + tags: ["document-rag", "graph-rag", "kgcore"], + } + + graph_store + document_store + agent + load + + graphrag_extract + kgcore + structured_store, + + // Structured RAG only + "structured": { + description: "Structured data extraction and querying", + tags: ["structured"], + } + + structured_store + structured_extract + agent + load, + + // Ontology RAG + knowledge cores + "ontology": { + description: "Ontology RAG + knowledge cores", + tags: ["onto-rag", "kgcore"], + } + + graph_store + ontorag_extract + agent + load + kgcore, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/graph-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/graph-store.jsonnet new file mode 100644 index 00000000..3cc8d2f0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/graph-store.jsonnet @@ -0,0 +1,75 @@ +// Graph store module +// Shared infrastructure for graph-based RAG (used by both GraphRAG and OntologyRAG) +// Handles knowledge graph storage, embeddings, and graph-based question answering + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local librarian_request = helpers.librarian_request; +local librarian_response = helpers.librarian_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with graph store configuration +llm_services + embeddings_service + { + + // External interfaces exposed by the graph store + "interfaces" +: { + // Data ingestion interfaces for graph construction + "entity-contexts-load": flow("entity-contexts-load:{id}"), + "triples-store": flow("triples-store:{id}"), + "graph-embeddings-store": flow("graph-embeddings-store:{id}"), + + // Query interfaces for graph-based operations + "graph-rag": request_response("graph-rag:{id}"), + "triples": request_response("triples:{id}"), + "graph-embeddings": request_response("graph-embeddings:{id}"), + }, + + // Flow-level processors - handle data streams for a specific flow instance + "flow" +: { + "graph-embeddings:{id}": { + input: flow("entity-contexts-load:{id}"), + output: flow("graph-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "triples-write:{id}": { + input: flow("triples-store:{id}"), + }, + "graph-embeddings-write:{id}": { + input: flow("graph-embeddings-store:{id}"), + }, + "graph-rag:{id}": { + request: request("graph-rag:{id}"), + response: response("graph-rag:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + "graph-embeddings-request": request("graph-embeddings:{id}"), + "graph-embeddings-response": response("graph-embeddings:{id}"), + "triples-request": request("triples:{id}"), + "triples-response": response("triples:{id}"), + explainability: flow("triples-store:{id}"), + "librarian-request": librarian_request, + "librarian-response": librarian_response, + }, + "triples-query:{id}": { + request: request("triples:{id}"), + response: response("triples:{id}"), + }, + "graph-embeddings-query:{id}": { + request: request("graph-embeddings:{id}"), + response: response("graph-embeddings:{id}"), + }, + }, + + // Blueprint-level processors - shared across all flow instances of this blueprint + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/graphrag-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/graphrag-extract.jsonnet new file mode 100644 index 00000000..6bcbca14 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/graphrag-extract.jsonnet @@ -0,0 +1,44 @@ +// GraphRAG extraction module +// Extraction method for GraphRAG - extracts definitions and relationships +// Mutually exclusive with OntologyRAG extraction (both write to graph store) + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts entity definitions from text chunks + // Identifies and defines key entities mentioned in the text + "kg-extract-definitions:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output definition triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition extraction prompts + "prompt-response": response("prompt:{id}"), + }, + + // Extracts relationships between entities + // Identifies how entities are connected and interact + "kg-extract-relationships:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output relationship triples + "prompt-request": request("prompt:{id}"), // Relationship extraction prompts + "prompt-response": response("prompt:{id}"), + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/helpers.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/helpers.jsonnet new file mode 100644 index 00000000..fc727329 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/helpers.jsonnet @@ -0,0 +1,39 @@ +// Helper functions for flow configuration +// Provides utility functions for constructing flow, request, and response URIs +// used throughout the TrustGraph flow configuration system + +// Creates a persistent flow URI for data streams +// Persistent flows retain messages until consumed +local flow(x) = "flow:tg:" + x; + +// Creates a non-persistent request URI for request-response patterns +// Non-persistent means messages are not retained if no consumer is present +local request(x) = "request:tg:" + x; + +// Creates a non-persistent response URI for request-response patterns +local response(x) = "response:tg:" + x; + +// State broadcast queue β€” persistent, last-value semantics +local state(x) = "state:tg:" + x; + +local librarian_request = request("librarian"); +local librarian_response = request("librarian"); + +// Creates a request-response pair for bidirectional communication +// Returns an object with both request and response URIs +local request_response(x) = { + request: request(x), + response: response(x), +}; + +// Export all helper functions for use in other modules +{ + flow: flow, + request: request, + response: response, + request_response: request_response, + state: state, + librarian_request: librarian_request, + librarian_response: librarian_response, +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/kgcore.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/kgcore.jsonnet new file mode 100644 index 00000000..bfc01ada --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/kgcore.jsonnet @@ -0,0 +1,31 @@ +// Knowledge Graph Core storage module +// Handles persistent storage of knowledge graph data +// Consolidates triples and graph embeddings into permanent storage +// Creates the core knowledge base for long-term use + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; + +{ + // No external interfaces - internal storage service + "interfaces" +: { + }, + + // No configurable parameters for core storage + "parameters" +: { + }, + + // Flow-level processors for knowledge graph storage + "flow" +: { + // Knowledge graph store consolidates extracted knowledge + // Takes processed triples and embeddings and stores them permanently + "kg-store:{id}": { + "triples-input": flow("triples-store:{id}"), // Input RDF triples stream + "graph-embeddings-input": flow("graph-embeddings-store:{id}"), // Input graph embeddings + }, + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/llm-parameters.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/llm-parameters.jsonnet new file mode 100644 index 00000000..a99f72ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/llm-parameters.jsonnet @@ -0,0 +1,59 @@ +{ + + // LLM model selection for normal LLM + "llm-model": { + "type": "llm-model", + "description": "LLM model", + "order": 1, + "advanced": false, + }, + + // LLM model for RAG operations + "llm-rag-model": { + "type": "llm-model", + "description": "LLM model for RAG", + "order": 2, + "advanced": true, + "controlled-by": "llm-model", + }, + + // LLM model selection for normal LLM + "llm-temperature": { + "type": "llm-temperature", + "description": "LLM temperature", + "order": 3, + "advanced": true, + }, + + // LLM model selection for normal LLM + "llm-rag-temperature": { + "type": "llm-temperature", + "description": "LLM temperature for RAG", + "order": 4, + "advanced": true, + }, + + "embeddings-model": { + "type": "embeddings-model", + "description": "Embeddings model", + "order": 5, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-size": { + "type": "chunk-size", + "description": "Chunk size", + "order": 6, + "advanced": true, + }, + + // LLM model selection for normal LLM + "chunk-overlap": { + "type": "chunk-overlap", + "description": "Chunk overlap", + "order": 7, + "advanced": true, + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/llm-services.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/llm-services.jsonnet new file mode 100644 index 00000000..ee6e09ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/llm-services.jsonnet @@ -0,0 +1,66 @@ +// Shared LLM services module +// Provides text completion, prompt processing, and metering services +// Import this module in any flow that requires LLM functionality + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local llm_parameters = import "llm-parameters.jsonnet"; + +{ + // Interfaces exposed by LLM services + "interfaces" +: { + "prompt": request_response("prompt:{id}"), + "text-completion": request_response("text-completion:{id}"), + }, + + // LLM configuration parameters + "parameters" +: llm_parameters, + + // Flow-level processors for LLM services + "flow" +: { + // Primary text completion service + "text-completion:{id}": { + request: request("text-completion:{id}"), + response: response("text-completion:{id}"), + model: "{llm-model}", + }, + + // RAG-specific text completion (may use different model) + "text-completion-rag:{id}": { + request: request("text-completion-rag:{id}"), + response: response("text-completion-rag:{id}"), + model: "{llm-rag-model}", + }, + + // Prompt processing service + "prompt:{id}": { + request: request("prompt:{id}"), + response: response("prompt:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + + // RAG-specific prompt processing + "prompt-rag:{id}": { + request: request("prompt-rag:{id}"), + response: response("prompt-rag:{id}"), + "text-completion-request": request("text-completion-rag:{id}"), + "text-completion-response": response("text-completion-rag:{id}"), + }, + + // Usage metering for primary completion + "metering:{id}": { + input: response("text-completion:{id}"), + }, + + // Usage metering for RAG completion + "metering-rag:{id}": { + input: response("text-completion-rag:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/load.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/load.jsonnet new file mode 100644 index 00000000..e51c568e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/load.jsonnet @@ -0,0 +1,53 @@ +// Document loading and preprocessing module +// Handles document ingestion, format conversion, and chunking +// Converts PDFs to text and splits documents into processable chunks + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; +local librarian_request = helpers.librarian_request; +local librarian_response = helpers.librarian_response; + +// Import shared services (load requires embeddings for chunk processing) +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with load-specific configuration +embeddings_service + { + + // External interfaces for document loading + "interfaces" +: { + "document-load": flow("document-load:{id}"), + "text-load": flow("text-document-load:{id}"), + }, + + // Flow-level processors for document preprocessing + "flow" +: { + // PDF decoder converts PDF documents to text + // Also emits page provenance triples and saves pages via librarian + "document-decoder:{id}": { + input: flow("document-load:{id}"), + output: flow("text-document-load:{id}"), + triples: flow("triples-store:{id}"), + "librarian-request": librarian_request, + "librarian-response": librarian_response, + }, + + // Chunker splits documents into smaller, processable pieces + // Also emits chunk provenance triples and saves chunks via librarian + "chunker:{id}": { + input: flow("text-document-load:{id}"), + output: flow("chunk-load:{id}"), + triples: flow("triples-store:{id}"), + "librarian-request": librarian_request, + "librarian-response": librarian_response, + "chunk-size": "{chunk-size}", + "chunk-overlap": "{chunk-overlap}", + }, + }, + + // Blueprint-level processors for document loading services + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/mcp-service.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/mcp-service.jsonnet new file mode 100644 index 00000000..5d599a67 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/mcp-service.jsonnet @@ -0,0 +1,31 @@ +// Shared MCP (Model Context Protocol) tool service module +// Provides MCP tool execution capabilities for agents +// Import this module in any flow that requires MCP tool integration + +local helpers = import "helpers.jsonnet"; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +{ + // Interfaces exposed by MCP service + "interfaces" +: { + "mcp-tool": request_response("mcp-tool:{id}"), + }, + + "parameters" +: { + }, + + // Flow-level processor for MCP tool execution + "flow" +: { + "mcp-tool:{id}": { + request: request("mcp-tool:{id}"), + response: response("mcp-tool:{id}"), + "text-completion-request": request("text-completion:{id}"), + "text-completion-response": response("text-completion:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/ontorag-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/ontorag-extract.jsonnet new file mode 100644 index 00000000..e5521d8a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/ontorag-extract.jsonnet @@ -0,0 +1,39 @@ +// OntologyRAG extraction module +// Extraction method for OntologyRAG - extracts using ontology definitions +// Mutually exclusive with GraphRAG extraction (both write to graph store) + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + // No external interfaces - this module provides internal extraction services + "interfaces" +: { + }, + + // No configurable parameters for basic KG extraction + "parameters" +: { + }, + + // Flow-level processors for knowledge extraction + "flow" +: { + // Extracts using ontology definitions + "kg-extract-ontology:{id}": { + input: flow("chunk-load:{id}"), // Input text chunks + triples: flow("triples-store:{id}"), // Output triples + "entity-contexts": flow("entity-contexts-load:{id}"), // Entity context information + "prompt-request": request("prompt:{id}"), // Definition + // extraction prompts + "prompt-response": response("prompt:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + + }, + + // No blueprint-level processors needed + "blueprint" +: { + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/structured-extract.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/structured-extract.jsonnet new file mode 100644 index 00000000..54f6a05d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/structured-extract.jsonnet @@ -0,0 +1,30 @@ +// Structured RAG extraction module +// Extracts structured rows from text chunks +// Outputs to rows-store for structured data querying + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; + +{ + "interfaces" +: { + }, + + "parameters" +: { + }, + + // Flow-level processor for structured row extraction + "flow" +: { + "kg-extract-rows:{id}": { + input: flow("chunk-load:{id}"), + output: flow("rows-store:{id}"), + "entity-contexts": flow("entity-contexts-load:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/structured-store.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/structured-store.jsonnet new file mode 100644 index 00000000..3668222d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/flows/structured-store.jsonnet @@ -0,0 +1,78 @@ +// Structured store module +// Shared infrastructure for structured data RAG +// Handles row storage, retrieval, and NLP query capabilities + +local helpers = import "helpers.jsonnet"; +local flow = helpers.flow; +local request = helpers.request; +local response = helpers.response; +local request_response = helpers.request_response; + +// Import shared services +local llm_services = import "llm-services.jsonnet"; +local embeddings_service = import "embeddings-service.jsonnet"; + +// Merge shared services with structured store configuration +llm_services + embeddings_service + { + + // External interfaces for structured store + "interfaces" +: { + // Row storage and querying + "rows-store": flow("rows-store:{id}"), + "row-embeddings-store": flow("row-embeddings-store:{id}"), + "rows": request_response("rows:{id}"), + "row-embeddings": request_response("row-embeddings:{id}"), + + // Query interfaces + "nlp-query": request_response("nlp-query:{id}"), + "structured-query": request_response("structured-query:{id}"), + "structured-diag": request_response("structured-diag:{id}"), + }, + + // Flow-level processors for structured storage and query + "flow" +: { + "row-embeddings:{id}": { + input: flow("rows-store:{id}"), + output: flow("row-embeddings-store:{id}"), + "embeddings-request": request("embeddings:{id}"), + "embeddings-response": response("embeddings:{id}"), + }, + "rows-write:{id}": { + input: flow("rows-store:{id}"), + }, + "row-embeddings-write:{id}": { + input: flow("row-embeddings-store:{id}"), + }, + "rows-query:{id}": { + request: request("rows:{id}"), + response: response("rows:{id}"), + }, + "row-embeddings-query:{id}": { + request: request("row-embeddings:{id}"), + response: response("row-embeddings:{id}"), + }, + "nlp-query:{id}": { + request: request("nlp-query:{id}"), + response: response("nlp-query:{id}"), + "prompt-request": request("prompt-rag:{id}"), + "prompt-response": response("prompt-rag:{id}"), + }, + "structured-query:{id}": { + request: request("structured-query:{id}"), + response: response("structured-query:{id}"), + "nlp-query-request": request("nlp-query:{id}"), + "nlp-query-response": response("nlp-query:{id}"), + "rows-query-request": request("rows:{id}"), + "rows-query-response": response("rows:{id}"), + }, + "structured-diag:{id}": { + request: request("structured-diag:{id}"), + response: response("structured-diag:{id}"), + "prompt-request": request("prompt:{id}"), + "prompt-response": response("prompt:{id}"), + }, + }, + + "blueprint" +: { + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/azure-openai.jsonnet new file mode 100644 index 00000000..c20ac7ae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/azure-openai.jsonnet @@ -0,0 +1,110 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure-openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-openai-" + key]:: value, + }, + +// Strategy is to specify the model with the AZURE_MODEL environment +// variable. This isn't something that can just be specified dynamically, +// it has to match what was provisioned in Azure. + + "azure-openai-max-output-tokens":: 4192, + "azure-openai-temperature":: 0.0, + "azure-openai-models":: models, + + "llm-models" +:: $["azure-openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + ] + $["pub-sub-args"] + [ + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-openai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_MODEL", "azure-model") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure-openai", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-openai-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/azure.jsonnet new file mode 100644 index 00000000..c8e1bcad --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/azure.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/azure.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["azure-" + key]:: value, + }, + + "azure-max-output-tokens":: 4096, + "azure-temperature":: 0.0, + "azure-models":: models, + + "llm-models" +:: $["azure-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + ] + $["pub-sub-args"] + [ + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("azure-ai-credentials") + .with_env_var("AZURE_TOKEN", "azure-token") + .with_env_var("AZURE_ENDPOINT", "azure-endpoint"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-azure", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "-x", + std.toString($["azure-max-output-tokens"]), + "-t", + "%0.3f" % $["azure-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/bedrock.jsonnet new file mode 100644 index 00000000..079531d8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/bedrock.jsonnet @@ -0,0 +1,102 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/bedrock.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["bedrock-" + key]:: value, + }, + + "bedrock-max-output-tokens":: 4096, + "bedrock-temperature":: 0.0, + "bedrock-models":: models, + + "llm-models" +:: $["bedrock-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + ] + $["pub-sub-args"] + [ + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("bedrock-credentials") + .with_env_var("AWS_ACCESS_KEY_ID", "aws-id-key") + .with_env_var("AWS_SECRET_ACCESS_KEY", "aws-secret") + .with_env_var("AWS_DEFAULT_REGION", "aws-region"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_bedrock) + .with_command([ + "text-completion-bedrock", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "-x", + std.toString($["bedrock-max-output-tokens"]), + "-t", + "%0.3f" % $["bedrock-temperature"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/claude.jsonnet new file mode 100644 index 00000000..7751c767 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/claude.jsonnet @@ -0,0 +1,102 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/claude.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["claude-" + key]:: value, + }, + + "claude-max-output-tokens":: 4096, + "claude-temperature":: 0.0, + "claude-models":: models, + + "llm-models" +:: $["claude-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + ] + $["pub-sub-args"] + [ + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("claude-credentials") + .with_env_var("CLAUDE_KEY", "claude-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-claude", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "-x", + std.toString($["claude-max-output-tokens"]), + "-t", + "%0.3f" % $["claude-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/cohere.jsonnet new file mode 100644 index 00000000..d6e2778b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/cohere.jsonnet @@ -0,0 +1,95 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/cohere.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["cohere-" + key]:: value, + }, + + "cohere-temperature":: 0.0, + "cohere-models":: models, + + "llm-models" +:: $["cohere-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + ] + $["pub-sub-args"] + [ + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("cohere-credentials") + .with_env_var("COHERE_KEY", "cohere-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-cohere", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "-t", + "%0.3f" % $["cohere-temperature"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/googleaistudio.jsonnet new file mode 100644 index 00000000..d2062384 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/googleaistudio.jsonnet @@ -0,0 +1,102 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/googleaistudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["googleaistudio-" + key]:: value, + }, + + "googleaistudio-max-output-tokens":: 4096, + "googleaistudio-temperature":: 0.0, + "googleaistudio-models":: models, + + "llm-models" +:: $["googleaistudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-googleaistudio", + ] + $["pub-sub-args"] + [ + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("googleaistudio-credentials") + .with_env_var("GOOGLE_AI_STUDIO_KEY", "googleaistudio-key"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-googleaistudio", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "-x", + std.toString($["googleaistudio-max-output-tokens"]), + "-t", + "%0.3f" % $["googleaistudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/llamafile.jsonnet new file mode 100644 index 00000000..3355404f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/llamafile.jsonnet @@ -0,0 +1,92 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/slm.jsonnet"; +local models = import "parameters/llamafile.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["llamafile-" + key]:: value, + }, + + "llamafile-models":: models, + + "llm-models" +:: $["llamafile-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("llamafile-credentials") + .with_env_var("LLAMAFILE_URL", "llamafile-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-llamafile", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/lmstudio.jsonnet new file mode 100644 index 00000000..bf59cced --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/lmstudio.jsonnet @@ -0,0 +1,102 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/lmstudio.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["lmstudio-" + key]:: value, + }, + + "lmstudio-max-output-tokens":: 4096, + "lmstudio-temperature":: 0.0, + "lmstudio-models":: models, + + "llm-models" +:: $["lmstudio-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + ] + $["pub-sub-args"] + [ + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("lmstudio-credentials") + .with_env_var("LMSTUDIO_URL", "lmstudio-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-lmstudio", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "-x", + std.toString($["lmstudio-max-output-tokens"]), + "-t", + "%0.3f" % $["lmstudio-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/mistral.jsonnet new file mode 100644 index 00000000..412620d2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/mistral.jsonnet @@ -0,0 +1,102 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/mistral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "mistral-max-output-tokens":: 4096, + "mistral-temperature":: 0.0, + "mistral-models":: models, + + "llm-models" +:: $["mistral-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + ] + $["pub-sub-args"] + [ + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-mistral", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "-x", + std.toString($["mistral-max-output-tokens"]), + "-t", + "%0.3f" % $["mistral-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/ollama.jsonnet new file mode 100644 index 00000000..b5bae965 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/ollama.jsonnet @@ -0,0 +1,100 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/ollama.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["ollama-" + key]:: value, + }, + + "ollama-models":: models, + + "llm-models" +:: $["ollama-models"], + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + ] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("ollama-credentials") + .with_env_var("OLLAMA_HOST", "ollama-host"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-ollama", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/openai.jsonnet new file mode 100644 index 00000000..1b72e6ec --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/openai.jsonnet @@ -0,0 +1,104 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/openai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["openai-" + key]:: value, + }, + + "openai-max-output-tokens":: 4096, + "openai-temperature":: 0.0, + "openai-models":: models, + + "llm-models" +:: $["openai-models"], + + "text-completion" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + ] + $["pub-sub-args"] + [ + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("openai-credentials") + .with_env_var("OPENAI_TOKEN", "openai-token") + .with_env_var("OPENAI_BASE_URL", "openai-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-openai", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "-x", + std.toString($["openai-max-output-tokens"]), + "-t", + "%0.3f" % $["openai-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/tgi.jsonnet new file mode 100644 index 00000000..0f9f058f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/tgi.jsonnet @@ -0,0 +1,106 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-" + key]:: value, + }, + + "tgi-max-output-tokens":: 1024, + "tgi-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + ] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("tgi-credentials") + .with_env_var("TGI_BASE_URL", "tgi-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-tgi", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["tgi-max-output-tokens"]), + "-t", + "%0.3f" % $["tgi-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/vertexai.jsonnet new file mode 100644 index 00000000..ce629029 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/vertexai.jsonnet @@ -0,0 +1,118 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vertexai.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vertexai-" + key]:: value, + }, + + "vertexai-private-key":: "/vertexai/private.json", + "vertexai-region":: "us-central1", + "vertexai-max-output-tokens":: 4096, + "vertexai-temperature":: 0.0, + "vertexai-models":: models, + + "llm-models" +:: $["vertexai-models"], + + "text-completion" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + ] + $["pub-sub-args"] + [ + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local cfgVol = engine.secretVolume( + "vertexai-creds", + "./vertexai", + { + "private.json": importstr "vertexai/private.json", + } + ); + + local container = + engine.container("text-completion-rag") + .with_image(images.trustgraph_vertexai) + .with_command([ + "text-completion-vertexai", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "-k", + $["vertexai-private-key"], + "-r", + $["vertexai-region"], + "-x", + std.toString($["vertexai-max-output-tokens"]), + "-t", + "%0.3f" % $["vertexai-temperature"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_volume_mount(cfgVol, "/vertexai"); + + local containerSet = engine.containers( + "text-completion-rag", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + cfgVol, + containerSet, + service, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/vllm.jsonnet new file mode 100644 index 00000000..e313548b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/llm/vllm.jsonnet @@ -0,0 +1,111 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local prompts = import "prompts/mixtral.jsonnet"; +local models = import "parameters/vllm.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-" + key]:: value, + }, + + "vllm-models":: models, + + "llm-models" +:: $["vllm-models"], + + "vllm-max-output-tokens":: 1024, + "vllm-temperature":: 0.0, + + "text-completion" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local container = + engine.container("text-completion") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + ] + $["pub-sub-args"] + [ + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "text-completion", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "text-completion-rag" +: { + + create:: function(engine) + + local concurrency = self.concurrency; + + local envSecrets = engine.envSecrets("vllm-credentials") + .with_env_var("VLLM_BASE_URL", "vllm-url"); + + local containerRag = + engine.container("text-completion-rag") + .with_image(images.trustgraph_flow) + .with_command([ + "text-completion-vllm", + ] + $["pub-sub-args"] + [ + "--id", + "text-completion-rag", + "--concurrency", + std.toString(concurrency), + "-x", + std.toString($["vllm-max-output-tokens"]), + "-t", + "%0.3f" % $["vllm-temperature"], + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSetRag = engine.containers( + "text-completion-rag", [ containerRag ] + ); + + local serviceRag = + engine.internalService(containerSetRag) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSetRag, + serviceRag, + ]) + + }, + +} + prompts + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/mcp/ddg-mcp-server.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/mcp/ddg-mcp-server.jsonnet new file mode 100644 index 00000000..04176f15 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/mcp/ddg-mcp-server.jsonnet @@ -0,0 +1,47 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "ddg-mcp-server-port":: 9870, + + "ddg-mcp-server" +: { + + create:: function(engine) + + local port = $["ddg-mcp-server-port"]; + + local container = + engine.container("ddg-mcp-server") + .with_image(images["ddg-mcp-server"]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(port, port, "mcp"); + + local containerSet = engine.containers( + "ddg-mcp-server", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(port, port, "mcp"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + mcp +:: { + "duckduckgo": { + "remote-name": "search", + local port = $["ddg-mcp-server-port"], + local url = "http://ddg-mcp-server:%s/mcp" % port, + "url": url, + } + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/cpu-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/cpu-tgi.jsonnet new file mode 100644 index 00000000..04b61566 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/cpu-tgi.jsonnet @@ -0,0 +1,68 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + "tgi-service-storage":: "20G", + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-cpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--cuda-graphs", + "0", + ]) + .with_environment({ + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-battlemage-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-battlemage-vllm.jsonnet new file mode 100644 index 00000000..7646398c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-battlemage-vllm.jsonnet @@ -0,0 +1,98 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-Nemo-Instruct-2407", + "vllm-service-cpus":: "32.0", + "vllm-service-memory":: "48G", + "vllm-service-storage":: "48G", + "vllm-service-tokenizer-mode":: "mistral", + "vllm-service-datatype":: "float16", + "vllm-service-quantization":: "woq_int4", + "vllm-service-hf-token":: null, + "vllm-service-max-model-len":: 8192, + "vllm-service-max-num-seqs":: 16, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-battlemage"]) + .with_entrypoint("") // Clear default entrypoint + .with_command([ + "python", + "-m", + "ipex_llm.vllm.xpu.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--device", + "xpu", + "--dtype", + $["vllm-service-datatype"], + "--enforce-eager", + "--max-model-len", + std.toString($["vllm-service-max-model-len"]), + "--max-num-seqs", + std.toString($["vllm-service-max-num-seqs"]), + "--load-in-low-bit", + $["vllm-service-quantization"], + "--trust-remote-code", + "--tokenizer-mode", + $["vllm-service-tokenizer-mode"], + "--disable-sliding-window", + ]) + .with_environment({ + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS: "1", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-gaudi-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-gaudi-tgi.jsonnet new file mode 100644 index 00000000..5af36b78 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-gaudi-tgi.jsonnet @@ -0,0 +1,96 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "meta-llama/Llama-3.3-70B-Instruct", + "tgi-service-cpus":: "64.0", + "tgi-service-memory":: "64G", + "tgi-service-storage":: "50G", + "tgi-service-num-shard":: 8, + "tgi-service-max-input-tokens":: 4096, + "tgi-service-max-total-tokens":: 4096, + "tgi-service-max-batch-size":: 128, + "tgi-service-max-concurrent-requests":: 512, + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-gaudi"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--sharded", + "true", + "--num-shard", + std.toString($["tgi-service-num-shard"]), + "--max-input-tokens", + std.toString($["tgi-service-max-input-tokens"]), + "--max-total-tokens", + std.toString($["tgi-service-max-total-tokens"]), + "--max-batch-size", + std.toString($["tgi-service-max-batch-size"]), + "--max-waiting-tokens", + "7", + "--max-concurrent-requests", + std.toString($["tgi-service-max-concurrent-requests"]), + "--cuda-graphs", + "0", + ]) + .with_runtime("habana") + .with_environment({ + HABANA_VISIBLE_DEVICES: "all", + OMPI_MCA_btl_vader_single_copy_mechanism: "none", + ENABLE_HPU_GRAPH: "true", + LIMIT_HPU_GRAPH: "true", + USE_FLASH_ATTENTION: "true", + FLASH_ATTENTION_RECOMPUTE: "true", + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-gaudi-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-gaudi-vllm.jsonnet new file mode 100644 index 00000000..81d31c14 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-gaudi-vllm.jsonnet @@ -0,0 +1,76 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "64.0", + "vllm-service-memory":: "64G", + "vllm-service-storage":: "50G", + "vllm-service-tensor-parallel-size":: 8, + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-gaudi"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--tensor-parallel-size", + std.toString($["vllm-service-tensor-parallel-size"]), + ]) + .with_runtime("habana") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + HABANA_VISIBLE_DEVICES: "all", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-xpu-tgi.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-xpu-tgi.jsonnet new file mode 100644 index 00000000..bc806874 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-xpu-tgi.jsonnet @@ -0,0 +1,75 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["tgi-service-" + key]:: value, + }, + + "tgi-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "tgi-service-cpus":: "8.0", + "tgi-service-memory":: "16G", + "tgi-service-storage":: "20G", + "tgi-service-hf-token":: null, + + "tgi-service" +: { + + create:: function(engine) + + local vol = engine.volume("tgi-storage") + .with_size($["tgi-service-storage"]); + + local container = + engine.container("tgi-service") + .with_image(images["tgi-service-intel-xpu"]) + .with_command([ + "--model-id", + $["tgi-service-model"], + "--hostname", + "0.0.0.0", + "--port", + "7000", + "--cuda-graphs", + "0", + ]) + .with_environment({ + } + ( + if $["tgi-service-hf-token"] != null + then { HF_TOKEN: $["tgi-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_group("video") + .with_group("render") + .with_capability("SYS_NICE") + .with_limits( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_reservations( + $["tgi-service-cpus"], $["tgi-service-memory"] + ) + .with_port(7000, 7000, "tgi") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "tgi-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "tgi"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-xpu-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-xpu-vllm.jsonnet new file mode 100644 index 00000000..f6e82905 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/intel-xpu-vllm.jsonnet @@ -0,0 +1,97 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "teknium/OpenHermes-2.5-Mistral-7B", + "vllm-service-cpus":: "8.0", + "vllm-service-memory":: "16G", + "vllm-service-storage":: "20G", + "vllm-service-datatype":: "float16", + "vllm-service-max-model-len":: 4096, + "vllm-service-max-num-seqs":: 16, + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-intel-xpu"]) + .with_command([ + "python", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + "--device", + "xpu", + "--dtype", + $["vllm-service-datatype"], + "--enforce-eager", + "--max-model-len", + std.toString($["vllm-service-max-model-len"]), + "--max-num-seqs", + std.toString($["vllm-service-max-num-seqs"]), + "--block-size", + "64", + "--gpu-memory-util", + "0.85", + "--trust-remote-code", + "--disable-sliding-window", + ]) + .with_environment({ + VLLM_USE_V1: "1", + VLLM_WORKER_MULTIPROC_METHOD: "spawn", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_device("/dev/dri", "/dev/dri") + .with_ipc("host") + .with_group("video") + .with_group("render") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_bind_mount("/dev/dri/by-path", "/dev/dri/by-path") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/nvidia-gpu-vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/nvidia-gpu-vllm.jsonnet new file mode 100644 index 00000000..d04876c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/model-hosting/nvidia-gpu-vllm.jsonnet @@ -0,0 +1,72 @@ +local images = import "values/images.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["vllm-service-" + key]:: value, + }, + + "vllm-service-model":: "mistralai/Mistral-7B-Instruct-v0.3", + "vllm-service-cpus":: "0.5", + "vllm-service-memory":: "1G", + "vllm-service-storage":: "50G", + "vllm-service-hf-token":: null, + + "vllm-service" +: { + + create:: function(engine) + + local vol = engine.volume("vllm-storage") + .with_size($["vllm-service-storage"]); + + local container = + engine.container("vllm-service") + .with_image(images["vllm-service-nvidia"]) + .with_command([ + "--model", + $["vllm-service-model"], + "--served-model-name", + "model", + "--host", + "0.0.0.0", + "--port", + "7000", + ]) + .with_runtime("nvidia") + .with_environment({ + VLLM_SKIP_WARMUP: "true", + } + ( + if $["vllm-service-hf-token"] != null + then { HF_TOKEN: $["vllm-service-hf-token"] } + else {} + )) + .with_privileged(true) + .with_ipc("host") + .with_capability("SYS_NICE") + .with_limits( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_reservations( + $["vllm-service-cpus"], $["vllm-service-memory"] + ) + .with_port(7000, 7000, "vllm") + .with_volume_mount(vol, "/root/.cache/huggingface"); + + local containerSet = engine.containers( + "vllm-service", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(7000, 7000, "vllm"); + + engine.resources([ + vol, + containerSet, + service, + ]) + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/monitoring/grafana.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/monitoring/grafana.jsonnet new file mode 100644 index 00000000..7ed6eadb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/monitoring/grafana.jsonnet @@ -0,0 +1,124 @@ +local images = import "values/images.jsonnet"; +local loki = import "loki.jsonnet"; + +{ + + "prometheus" +: { + + create:: function(engine) + + local vol = engine.volume("prometheus-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "prometheus-cfg", "prometheus", + { + "prometheus.yml": $["prometheus-config"], + } + ); + + local container = + engine.container("prometheus") + .with_image(images.prometheus) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M") + .with_port(9090, 9090, "http") + .with_volume_mount(cfgVol, "/etc/prometheus/") + .with_volume_mount(vol, "/prometheus"); + + local containerSet = engine.containers( + "prometheus", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9090, 9090, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + + "grafana" +: { + + create:: function(engine) + + local vol = engine.volume("grafana-storage").with_size("20G"); + + local provDashVol = engine.configVolume( + "prov-dash", "grafana/provisioning/", + { + "dashboard.yml": + importstr "grafana/provisioning/dashboard.yml", + } + + ); + + local provDataVol = engine.configVolume( + "prov-data", "grafana/provisioning/", + { + "datasource.yml": + importstr "grafana/provisioning/datasource.yml", + } + + ); + + local dashVol = engine.configVolume( + "dashboards", "grafana/dashboards/", + { + "overview-dashboard.json": + $["overview-dashboard"], + "log-dashboard.json": + importstr "grafana/dashboards/log-dashboard.json", + } + + ); + + local container = + engine.container("grafana") + .with_image(images.grafana) + .with_environment({ + // GF_AUTH_ANONYMOUS_ORG_ROLE: "Admin", + // GF_AUTH_ANONYMOUS_ENABLED: "true", + // GF_ORG_ROLE: "Admin", + GF_ORG_NAME: "trustgraph.ai", + // GF_SERVER_ROOT_URL: "https://example.com", + }) + .with_limits("1.0", "256M") + .with_reservations("0.5", "256M") + .with_port(3000, 3000, "cassandra") + .with_volume_mount(vol, "/var/lib/grafana") + .with_volume_mount( + provDashVol, "/etc/grafana/provisioning/dashboards/" + ) + .with_volume_mount( + provDataVol, "/etc/grafana/provisioning/datasources/" + ) + .with_volume_mount( + dashVol, "/var/lib/grafana/dashboards/" + ); + + local containerSet = engine.containers( + "grafana", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3000, 3000, "http"); + + engine.resources([ + vol, + provDashVol, + provDataVol, + dashVol, + containerSet, + service, + ]) + + }, + +} + loki + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/monitoring/loki.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/monitoring/loki.jsonnet new file mode 100644 index 00000000..0774f6cf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/monitoring/loki.jsonnet @@ -0,0 +1,45 @@ +local images = import "values/images.jsonnet"; + +{ + + "loki" +: { + + create:: function(engine) + + local vol = engine.volume("loki-data").with_size("20G"); + + local cfgVol = engine.configVolume( + "loki-cfg", "loki", + { + "local-config.yaml": importstr "loki/local-config.yaml", + } + ); + + local container = + engine.container("loki") + .with_image(images.loki) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M") + .with_port(3100, 3100, "http") + .with_volume_mount(cfgVol, "/etc/loki/") + .with_volume_mount(vol, "/loki"); + + local containerSet = engine.containers( + "loki", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(3100, 3100, "http"); + + engine.resources([ + cfgVol, + vol, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/ocr/mistral-ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/ocr/mistral-ocr.jsonnet new file mode 100644 index 00000000..cf8085ff --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/ocr/mistral-ocr.jsonnet @@ -0,0 +1,48 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + with:: function(key, value) + self + { + ["mistral-" + key]:: value, + }, + + "document-decoder" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("mistral-credentials") + .with_env_var("MISTRAL_TOKEN", "mistral-token"); + + local container = + engine.container("mistral-ocr") + .with_image(images.trustgraph_flow) + .with_command([ + "pdf-ocr-mistral", + ] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "mistral-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/ocr/ocr.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/ocr/ocr.jsonnet new file mode 100644 index 00000000..8f103198 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/ocr/ocr.jsonnet @@ -0,0 +1,36 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +{ + + "document-decoder" +: { + + create:: function(engine) + + local container = + engine.container("pdf-ocr") + .with_image(images.trustgraph_ocr) + .with_command([ + "pdf-ocr", + ] + $["pub-sub-args"] + [ + ]) + .with_limits("1.0", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "pdf-ocr", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/azure-openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/azure-openai.jsonnet new file mode 100644 index 00000000..dcf706d4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/azure-openai.jsonnet @@ -0,0 +1,9 @@ +// Azure OpenAI LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-5.2", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/azure.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/azure.jsonnet new file mode 100644 index 00000000..6ffa5fdf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/azure.jsonnet @@ -0,0 +1,9 @@ +// Azure LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/bedrock.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/bedrock.jsonnet new file mode 100644 index 00000000..6c2c7e90 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/bedrock.jsonnet @@ -0,0 +1,84 @@ +// AWS Bedrock LLM Model Definitions +// Defines available models and their configurations for AWS Bedrock + +{ + "type": "string", + "description": "LLM model to use", + "default": "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + "enum": [ + // ── Anthropic Claude ────────────────────────────────────── + { + id: "global.anthropic.claude-opus-4-6-v1", + description: "Claude Opus 4.6 (maximum intelligence)" + }, + { + id: "global.anthropic.claude-opus-4-5-20251101-v1:0", + description: "Claude Opus 4.5 (frontier coding + agents)" + }, + { + id: "global.anthropic.claude-sonnet-4-5-20250929-v1:0", + description: "Claude Sonnet 4.5 (complex agents + coding)" + }, + { + id: "global.anthropic.claude-haiku-4-5-20251001-v1:0", + description: "Claude Haiku 4.5 (fastest with near-frontier intelligence)" + }, + { + id: "global.anthropic.claude-opus-4-1-20250805-v1:0", + description: "Claude Opus 4.1 (agentic search + reasoning)" + }, + { + id: "global.anthropic.claude-sonnet-4-20250514-v1:0", + description: "Claude Sonnet 4.0" + }, + { + id: "anthropic.claude-3-5-haiku-20241022-v1:0", + description: "Claude 3.5 Haiku (legacy)" + }, + + // ── Meta Llama ──────────────────────────────────────────── + { + id: "us.meta.llama4-maverick-17b-instruct-v1:0", + description: "Llama 4 Maverick 17B (128 experts, 400B params, multimodal)" + }, + { + id: "us.meta.llama4-scout-17b-instruct-v1:0", + description: "Llama 4 Scout 17B (16 experts, 3.5M context)" + }, + { + id: "us.meta.llama3-3-70b-instruct-v1:0", + description: "Llama 3.3 70B Instruct" + }, + + // ── Mistral AI ──────────────────────────────────────────── + { + id: "us.mistral.mistral-large-2511-v1:0", + description: "Mistral Large 3 (flagship text, 128K context)" + }, + { + id: "us.mistral.magistral-small-2506-v1:0", + description: "Magistral Small 1.2 (reasoning, cost-effective)" + }, + + // ── DeepSeek ────────────────────────────────────────────── + { + id: "us.deepseek.r1-v1:0", + description: "DeepSeek-R1 (reasoning)" + }, + + // ── Amazon Nova ─────────────────────────────────────────── + { + id: "us.amazon.nova-pro-v1:0", + description: "Amazon Nova Pro (multimodal, balanced)" + }, + { + id: "us.amazon.nova-lite-v1:0", + description: "Amazon Nova Lite (fast, multimodal)" + }, + { + id: "us.amazon.nova-micro-v1:0", + description: "Amazon Nova Micro (text-only, cheapest)" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/chunking-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/chunking-param-types.jsonnet new file mode 100644 index 00000000..f82fa3f7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/chunking-param-types.jsonnet @@ -0,0 +1,25 @@ +// Chunk parameter type definitions + +{ + "chunk-size": { + "type": "integer", + "description": "Chunk size", + "placeholder": 2000, + "helper": "An integer, usually 2000 .. 8000", + "default": 2000, + "min": 0, + "max": 32768, + "required": true + }, + "chunk-overlap": { + "type": "integer", + "description": "Chunk overlap", + "placeholder": 50, + "helper": "An integer, usually 50 .. 100", + "default": 50, + "min": 0, + "max": 8000, + "required": true + }, +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/claude.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/claude.jsonnet new file mode 100644 index 00000000..b14a317c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/claude.jsonnet @@ -0,0 +1,39 @@ +// Claude LLM Model Definitions +// Defines available models and their configurations for Anthropic's Claude + +{ + "type": "string", + "description": "LLM model to use", + "default": "claude-sonnet-4-5-20250929", + "enum": [ + { + id: "claude-opus-4-6", + description: "Claude Opus 4.6 (maximum intelligence)" + }, + { + id: "claude-opus-4-5-20251101", + description: "Claude Opus 4.5 (frontier coding + agents)" + }, + { + id: "claude-sonnet-4-5-20250929", + description: "Claude Sonnet 4.5 (complex agents + coding)" + }, + { + id: "claude-haiku-4-5-20251001", + description: "Claude Haiku 4.5 (fast)" + }, + { + id: "claude-opus-4-1-20250805", + description: "Claude Opus 4.1 (agentic search + reasoning)" + }, + { + id: "claude-sonnet-4-20250514", + description: "Claude Sonnet 4.0" + }, + { + id: "claude-3-5-haiku-20241022", + description: "Claude 3.5 Haiku" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/cohere.jsonnet new file mode 100644 index 00000000..5a5865a3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/cohere.jsonnet @@ -0,0 +1,35 @@ +// Cohere LLM Model Definitions +// Defines available models and their configurations for Cohere + +{ + "type": "string", + "description": "LLM model to use", + "default": "command-r-plus-08-2024", + "enum": [ + { + id: "command-r-plus-08-2024", + description: "Command R+ (August 2024)" + }, + { + id: "command-r-08-2024", + description: "Command R (August 2024)" + }, + { + id: "command-r-plus", + description: "Command R+ (legacy)" + }, + { + id: "command-r", + description: "Command R (legacy)" + }, + { + id: "command", + description: "Command" + }, + { + id: "command-light", + description: "Command Light" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/embeddings-fastembed.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/embeddings-fastembed.jsonnet new file mode 100644 index 00000000..477e25dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/embeddings-fastembed.jsonnet @@ -0,0 +1,127 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "sentence-transformers/all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "BAAI/bge-small-en-v1.5", + "description": "bge-small-en-v1.5" + }, + { + "id": "BAAI/bge-small-zh-v1.5", + "description": "bge-small-zh-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-xs", + "description": "snowflake-arctic-embed-xs" + }, + { + "id": "jinaai/jina-embeddings-v2-small-en", + "description": "jina-embeddings-v2-small-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "snowflake/snowflake-arctic-embed-s", + "description": "snowflake-arctic-embed-s" + }, + { + "id": "BAAI/bge-small-en", + "description": "bge-small-en" + }, + { + "id": "BAAI/bge-base-en-v1.5", + "description": "bge-base-en-v1.5" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", + "description": "paraphrase-multilingual-MiniLM-L12-v2" + }, + { + "id": "Qdrant/clip-ViT-B-32-text", + "description": "clip-ViT-B-32-text" + }, + { + "id": "jinaai/jina-embeddings-v2-base-de", + "description": "jina-embeddings-v2-base-de" + }, + { + "id": "BAAI/bge-base-en", + "description": "bge-base-en" + }, + { + "id": "snowflake/snowflake-arctic-embed-m", + "description": "snowflake-arctic-embed-m" + }, + { + "id": "thenlper/gte-base", + "description": "gte-base" + }, + { + "id": "jinaai/jina-embeddings-v2-base-en", + "description": "jina-embeddings-v2-base-en" + }, + { + "id": "nomic-ai/nomic-embed-text-v1", + "description": "nomic-embed-text-v1" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5", + "description": "nomic-embed-text-v1.5" + }, + { + "id": "snowflake/snowflake-arctic-embed-m-long", + "description": "snowflake-arctic-embed-m-long" + }, + { + "id": "jinaai/jina-clip-v1", + "description": "jina-clip-v1" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + { + "id": "jinaai/jina-embeddings-v2-base-es", + "description": "jina-embeddings-v2-base-es" + }, + { + "id": "jinaai/jina-embeddings-v2-base-code", + "description": "jina-embeddings-v2-base-code" + }, + { + "id": "jinaai/jina-embeddings-v2-base-zh", + "description": "jina-embeddings-v2-base-zh" + }, + { + "id": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + "description": "paraphrase-multilingual-mpnet-base-v2" + }, + { + "id": "snowflake/snowflake-arctic-embed-l", + "description": "snowflake-arctic-embed-l" + }, + { + "id": "BAAI/bge-large-en-v1.5", + "description": "bge-large-en-v1.5" + }, + { + "id": "thenlper/gte-large", + "description": "gte-large" + }, + { + "id": "intfloat/multilingual-e5-large", + "description": "multilingual-e5-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/embeddings-huggingface.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/embeddings-huggingface.jsonnet new file mode 100644 index 00000000..32254a2b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/embeddings-huggingface.jsonnet @@ -0,0 +1,31 @@ +// Embeddings model definitions for fastembed +// Defines available models and their configurations for Fastembed + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "sentence-transformers/all-MiniLM-L6-v2", + "enum": [ + { + "id": "all-MiniLM-L6-v2", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "all-mpnet-base-v2", + "description": "all-mpnet-base-v2" + }, + { + "id": "all-distilroberta-v1", + "description": "all-distilroberta-v1" + }, + { + "id": "stsb-bert-large", + "description": "stsb-bert-large" + }, + { + "id": "sentence-camembert-large", + "description": "sentence-camembert-large" + } + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/embeddings-ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/embeddings-ollama.jsonnet new file mode 100644 index 00000000..c3f29fbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/embeddings-ollama.jsonnet @@ -0,0 +1,23 @@ +// Embeddings model definitions for Ollama +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "Embeddings model to use", + "default": "all-minilm:latest", + "enum": [ + { + "id": "all-minilm:latest", + "description": "all-MiniLM-L6-v2" + }, + { + "id": "nomic-ai/nomic-embed-text-v1.5-Q", + "description": "nomic-embed-text-v1.5-Q" + }, + { + "id": "mixedbread-ai/mxbai-embed-large-v1", + "description": "mxbai-embed-large-v1" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/googleaistudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/googleaistudio.jsonnet new file mode 100644 index 00000000..6d454ea7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/googleaistudio.jsonnet @@ -0,0 +1,54 @@ +// Google AI Studio LLM Model Definitions +// Defines available models and their configurations for Google AI Studio + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 3 models (preview) + { + id: "gemini-3-pro-preview", + description: "Gemini 3 Pro (preview)" + }, + { + id: "gemini-3-flash-preview", + description: "Gemini 3 Flash (preview)" + }, + + // Gemini 2.5 models + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-001", + description: "Gemini 2.0 Flash" + }, + { + id: "gemini-2.0-flash-lite-001", + description: "Gemini 2.0 Flash Lite" + }, + + // Gemma models + { + id: "gemma-3-27b", + description: "Gemma 3 27B" + }, + { + id: "gemma-3n-e4b", + description: "Gemma 3n E4B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/llamafile.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/llamafile.jsonnet new file mode 100644 index 00000000..f6480aa9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/llamafile.jsonnet @@ -0,0 +1,9 @@ +// LlamaFile LLM Model Definitions +// Model input is just text + +{ + "type": "string", + "description": "LLM model to use", + "default": "phi4:14b", + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/lmstudio.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/lmstudio.jsonnet new file mode 100644 index 00000000..1c88da3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/lmstudio.jsonnet @@ -0,0 +1,150 @@ +// LMStudio LLM Model Definitions +// Defines available models and their configurations for LMStudio + +{ + "type": "string", + "description": "LLM model to use", + "default": "llama3.1:70b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/mistral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/mistral.jsonnet new file mode 100644 index 00000000..07f75b81 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/mistral.jsonnet @@ -0,0 +1,61 @@ +// Mistral LLM Model Definitions +// Defines available models and their configurations for Mistral AI + +{ + "type": "string", + "description": "LLM model to use", + "default": "mistral-medium-2508", + "enum": [ + // Featured models + { + id: "mistral-medium-2508", + description: "Mistral Medium 3.1" + }, + { + id: "mistral-large-2512", + description: "Mistral Large 3" + }, + { + id: "mistral-small-2506", + description: "Mistral Small 3.2" + }, + { + id: "ministral-14b-2512", + description: "Ministral 3 14B" + }, + { + id: "ministral-8b-2512", + description: "Ministral 3 8B" + }, + { + id: "ministral-3b-2512", + description: "Ministral 3 3B" + }, + { + id: "magistral-medium-2509", + description: "Magistral Medium 1.2 (reasoning)" + }, + { + id: "magistral-small-2509", + description: "Magistral Small 1.2 (reasoning)" + }, + { + id: "devstral-2512", + description: "Devstral 2 (code)" + }, + // Other models + { + id: "codestral-2508", + description: "Codestral (code)" + }, + { + id: "pixtral-large-2411", + description: "Pixtral Large (vision)" + }, + { + id: "open-mistral-nemo", + description: "Open Mistral Nemo" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/ollama.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/ollama.jsonnet new file mode 100644 index 00000000..4ef98a6a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/ollama.jsonnet @@ -0,0 +1,150 @@ +// Ollama LLM Model Definitions +// Defines available models and their configurations for Ollama + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemma3:12b", + "enum": [ + // Gemma3 models + { + id: "gemma3:1b", + description: "Gemma3 1B" + }, + { + id: "gemma3:4b", + description: "Gemma3 4B" + }, + { + id: "gemma3:12b", + description: "Gemma3 12B" + }, + { + id: "gemma3:27b", + description: "Gemma3 27B" + }, + + // Phi4 models + { + id: "phi4:mini:3.8b", + description: "Phi4 Mini 3.8B" + }, + { + id: "phi4:14b", + description: "Phi4 14B" + }, + + // DeepSeek-R1 models + { + id: "deepseek-r1:671b", + description: "DeepSeek-R1 671B" + }, + { + id: "deepseek-r1:70b", + description: "DeepSeek-R1 70B" + }, + { + id: "deepseek-r1:32b", + description: "DeepSeek-R1 32B" + }, + { + id: "deepseek-r1:14b", + description: "DeepSeek-R1 14B" + }, + { + id: "deepseek-r1:8b", + description: "DeepSeek-R1 8B" + }, + { + id: "deepseek-r1:7b", + description: "DeepSeek-R1 7B" + }, + { + id: "deepseek-r1:1.5b", + description: "DeepSeek-R1 1.5B" + }, + + // Llama3.1 models + { + id: "llama3.1:405b", + description: "Llama 3.1 405B" + }, + { + id: "llama3.1:70b", + description: "Llama 3.1 70B" + }, + { + id: "llama3.1:8b", + description: "Llama 3.1 8B" + }, + + // Gemma2 models + { + id: "gemma2:2b", + description: "Gemma2 2B" + }, + { + id: "gemma2:9b", + description: "Gemma2 9B" + }, + { + id: "gemma2:27b", + description: "Gemma2 27B" + }, + + // Qwen2.5 models + { + id: "qwen2.5:0.5b", + description: "Qwen2.5 0.5B" + }, + { + id: "qwen2.5:1.5b", + description: "Qwen2.5 1.5B" + }, + { + id: "qwen2.5:3b", + description: "Qwen2.5 3B" + }, + { + id: "qwen2.5:7b", + description: "Qwen2.5 7B" + }, + { + id: "qwen2.5:14b", + description: "Qwen2.5 14B" + }, + { + id: "qwen2.5:32b", + description: "Qwen2.5 32B" + }, + { + id: "qwen2.5:72b", + description: "Qwen2.5 72B" + }, + + // Mistral models + { + id: "mistral:7b", + description: "Mistral 7B" + }, + { + id: "mistral-nemo:12b", + description: "Mistral Nemo 12B" + }, + { + id: "mistral-large:123b", + description: "Mistral Large 123B" + }, + + // Command-R models + { + id: "command-r:35b", + description: "Command-R 35B" + }, + { + id: "command-r-plus:104b", + description: "Command-R Plus 104B" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/openai.jsonnet new file mode 100644 index 00000000..5039c982 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/openai.jsonnet @@ -0,0 +1,51 @@ +// OpenAI LLM Model Definitions +// Defines available models and their configurations for OpenAI's platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gpt-5.2", + "enum": [ + { + id: "gpt-5.2-pro", + description: "GPT-5.2 Pro (maximum intelligence, slow)" + }, + { + id: "gpt-5.2", + description: "GPT-5.2 (flagship, best general-purpose)" + }, + { + id: "gpt-5.2-codex", + description: "GPT-5.2 Codex (optimized for agentic coding)" + }, + { + id: "gpt-5.1", + description: "GPT-5.1 (previous flagship)" + }, + { + id: "gpt-5", + description: "GPT-5 (previous gen reasoning)" + }, + { + id: "gpt-4.1", + description: "GPT-4.1 (coding + long context, 1M tokens)" + }, + { + id: "gpt-4.1-mini", + description: "GPT-4.1 Mini (fast + affordable)" + }, + { + id: "gpt-4.1-nano", + description: "GPT-4.1 Nano (ultra-fast, cheapest)" + }, + { + id: "gpt-4o", + description: "GPT-4o (legacy multimodal)" + }, + { + id: "gpt-4o-mini", + description: "GPT-4o Mini (legacy, budget)" + }, + ], + "required": true +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/temperature-param-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/temperature-param-types.jsonnet new file mode 100644 index 00000000..f5523e88 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/temperature-param-types.jsonnet @@ -0,0 +1,15 @@ +// LLM temperature definitions + +{ + "llm-temperature": { + "type": "float", + "description": "LLM temperature", + "placeholder": 0.3, + "helper": "A floating point number between 0 and 1", + "default": 0.3, + "min": 0.0, + "max": 10.0, + "required": true + } +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/vertexai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/vertexai.jsonnet new file mode 100644 index 00000000..34318967 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/vertexai.jsonnet @@ -0,0 +1,81 @@ +// VertexAI LLM Model Definitions +// Defines available models and their configurations for Google's VertexAI platform + +{ + "type": "string", + "description": "LLM model to use", + "default": "gemini-2.5-flash-lite", + "enum": [ + // Gemini 3 models (preview) + { + id: "gemini-3-pro-preview", + description: "Gemini 3 Pro (preview)" + }, + { + id: "gemini-3-flash-preview", + description: "Gemini 3 Flash (preview)" + }, + + // Gemini 2.5 models + { + id: "gemini-2.5-pro", + description: "Gemini 2.5 Pro" + }, + { + id: "gemini-2.5-flash", + description: "Gemini 2.5 Flash" + }, + { + id: "gemini-2.5-flash-lite", + description: "Gemini 2.5 Flash Lite" + }, + + // Gemini 2.0 models + { + id: "gemini-2.0-flash-001", + description: "Gemini 2.0 Flash" + }, + { + id: "gemini-2.0-flash-lite-001", + description: "Gemini 2.0 Flash Lite" + }, + + // Gemma models + { + id: "gemma-3-27b", + description: "Gemma 3 27B" + }, + { + id: "gemma-3n-e4b", + description: "Gemma 3n E4B" + }, + + // Claude models on VertexAI + { + id: "claude-opus-4-6", + description: "Claude Opus 4.6" + }, + { + id: "claude-opus-4-5", + description: "Claude Opus 4.5" + }, + { + id: "claude-sonnet-4-5", + description: "Claude Sonnet 4.5" + }, + { + id: "claude-haiku-4-5", + description: "Claude Haiku 4.5" + }, + { + id: "claude-opus-4-1", + description: "Claude Opus 4.1" + }, + { + id: "claude-sonnet-4", + description: "Claude Sonnet 4" + }, + ], + "required": true +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/vllm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/vllm.jsonnet new file mode 100644 index 00000000..120342b2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/parameters/vllm.jsonnet @@ -0,0 +1,18 @@ +// vLLM Model Definitions +// Defines available models and their configurations for vLLM +// vLLM works with any HuggingFace model. We have to use the model +// vLLM was initialised with + +{ + "type": "string", + "description": "LLM model to use", + "default": "model", + "enum": [ + // Llama 3.1 models + { + id: "model", + description: "Pre-defined model" + }, + ], + "required": true +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/profiles/memory-profile-low.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/profiles/memory-profile-low.jsonnet new file mode 100644 index 00000000..7a6829e3 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/profiles/memory-profile-low.jsonnet @@ -0,0 +1,169 @@ +// Low memory profile - reduces memory allocation across components. +// Include this at the END of your configuration to override default memory +// settings with lower values suitable for memory-constrained environments. +// +// Usage: {"name": "memory-profile-low", "parameters": {}} +// +// Note: This trades some performance headroom for reduced memory usage. +// Monitor for OOM errors under heavy load. + +{ + + // Override Pulsar stack memory settings + "pulsar" +: { + + // Zookeeper: 512M -> 300M + "zk-memory-limit":: "300M", + "zk-memory-reservation":: "200M", + "zk-heap":: "128m", + "zk-direct-memory":: "64m", + + // Bookie: 1024M -> 600M + "bookie-memory-limit":: "600M", + "bookie-memory-reservation":: "400M", + "bookie-heap":: "128m", + "bookie-direct-memory":: "128m", + + // Broker: 800M -> 512M + "broker-memory-limit":: "512M", + "broker-memory-reservation":: "400M", + "broker-heap":: "192m", + "broker-direct-memory":: "192m", + + // Pulsar-init: 256M -> 128M + "init-memory-limit":: "128M", + "init-memory-reservation":: "128M", + "init-heap":: "64m", + "init-direct-memory":: "64m", + + }, + + // Override Cassandra memory settings: 1000M -> 600M + "cassandra" +: { + "memory-limit":: "600M", + "memory-reservation":: "500M", + "heap":: "200M", + }, + + // Override Qdrant memory settings: 1024M -> 600M + // Also enables mmap for vectors/payloads (trades latency for memory) + "qdrant" +: { + "memory-limit":: "600M", + "memory-reservation":: "500M", + "memmap-threshold-kb":: "1", + "on-disk-payload":: "true", + }, + + // TrustGraph core services - 50% memory reservations + "api-gateway" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "chunker" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "config-svc" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-decoder" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "mcp-tool" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "mcp-server" +: { + "memory-reservation":: "128M", // 256M -> 128M + }, + + "metering" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "metering-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-store" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-manager" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "prompt" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "prompt-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "document-embeddings" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + "librarian" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "agent-manager" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + // Graph RAG services + "kg-extract-definitions" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-relationships" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-agent" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "kg-extract-ontology" +: { + "memory-reservation":: "150M", // 300M -> 150M + }, + + "kg-extract-objects" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "graph-rag" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "graph-embeddings" +: { + "memory-reservation":: "256M", // 512M -> 256M + }, + + // Structured data services + "nlp-query" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "structured-query" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + + "structured-diag" +: { + "memory-reservation":: "48M", // 96M -> 48M + }, + + // Init service + "init-trustgraph" +: { + "memory-reservation":: "64M", // 128M -> 64M + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/agent-kg-extract.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/agent-kg-extract.txt new file mode 100644 index 00000000..36dbd74f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/agent-kg-extract.txt @@ -0,0 +1,28 @@ +Analyze the following text and extract both entity definitions and relationships. + +For definitions, extract entities and their explanations or descriptions. +For relationships, extract subject-predicate-object triples where subjects and objects are entities, and predicates are relationship types. + +Text: {{text}} + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include a "type" field to distinguish between definitions and relationships. + +For definitions, output: +{"type": "definition", "entity": "entity_name", "definition": "definition_text"} + +For relationships, output: +{"type": "relationship", "subject": "subject_entity", "predicate": "relationship_type", "object": "object_entity_or_literal", "object-entity": true} + +Requirements: +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations, only output JSONL + +Example output: +{"type": "definition", "entity": "DNA", "definition": "Deoxyribonucleic acid, a molecule carrying genetic instructions"} +{"type": "definition", "entity": "RNA", "definition": "Ribonucleic acid, essential for coding and gene expression"} +{"type": "relationship", "subject": "DNA", "predicate": "transcribes_to", "object": "RNA", "object-entity": true} +{"type": "relationship", "subject": "DNA", "predicate": "located_in", "object": "cell nucleus", "object-entity": true} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/agent-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/agent-prompt.txt new file mode 100644 index 00000000..9155d20f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/agent-prompt.txt @@ -0,0 +1,104 @@ +# ReAct Agent System Prompt + +You are an AI assistant that uses the ReAct (Reasoning + Acting) framework to solve problems through systematic reasoning and tool use. + +## Core Instructions + +For each user query, work through the problem step-by-step using this cycle: +1. **Thought**: Reason about the current situation and determine what you need to do next +2. **Action**: Take ONE specific action using an available tool +3. Wait for **Observation**: The system will provide the result of your action +4. Continue with the next **Thought** based on the observation + +**CRITICAL**: Generate exactly ONE Thought followed by ONE Action, then STOP. Do not generate multiple Thought/Action pairs in a single response. Do not generate Observations yourself - the system will provide them. + +## Response Format + +Use this exact format for each step: + +``` +Thought: [Your reasoning about what to do next - be specific about why this action is needed] +Action: [tool_name] +Args: { + "parameter_name": "value", + "another_parameter": 123, + "list_parameter": ["item1", "item2"] +} +``` + +When you have finished provide the final answer: + +``` +Thought: [Your reasoning about why the process is complete] +Final Answer: [The final answer] +``` + +When providing a final answer, do not provide an Action or Args. + +## Action Format Rules + +1. **Tool Name**: Write "Action: " followed by the exact tool name on its own line +2. **Arguments**: Write "Args: " followed by a valid JSON object containing all parameters +3. **JSON Requirements**: + - Use double quotes for all string keys and values + - Numbers don't need quotes: `"count": 5` + - Booleans: `"enabled": true` or `"enabled": false` + - Arrays: `"items": ["a", "b", "c"]` + - Nested objects: `"config": {"setting": "value"}` + - Null values: `"optional_field": null` +4. **Required Parameters**: Include all required parameters for the tool +5. **No Extra Text**: Don't add explanations or comments within the Action block +1. **Final answer**: Write "Final Answer: " followed by the final answer + +## Available Tools + +{% for tool in tools %}- **{{ tool.name }}**: {{ tool.description }} +{% for arg in tool.arguments %} - Required: `"{{ arg.name }}"` ({{ arg.type }}): {{ arg.description }} +{% endfor %} +{% endfor %} + +## Behavior Rules + +1. **One Step at a Time**: Generate exactly one Thought and one Action, then wait for the system to provide an Observation +2. **Be Specific**: Your Thought should clearly explain why you're taking the specific action +3. **Use Context**: Build on previous Observations to inform your next steps +4. **Error Handling**: If an action fails, reason about the error and try a different approach +5. **Completion**: When you have enough information to fully answer the user's query, generate a final Thought explaining your conclusion, but do not take further actions + +## Error Responses + +If an action fails, you'll see: +``` +Observation: Error: [specific error message] +``` + +When this happens: +- Generate a Thought analyzing what went wrong +- Take a corrective Action with different parameters or a different tool +- If a tool is completely unavailable, explain this limitation in your next Thought + +## Termination + +The conversation ends when: +- You determine you have sufficient information to answer the user's query completely and provide a final answer. +- You encounter an unrecoverable error that prevents task completion +- The system reaches the maximum iteration limit + +## Important Notes + +- **Never generate Observations yourself** - only the system provides these +- **Always validate your JSON** - malformed JSON will cause action failures +- **Stay focused** - each Thought should directly relate to solving the user's query +- **Be efficient** - choose actions that gather the most relevant information for the task + +# Proceed + +Question: {{question}} + +{% for h in history %} +Action: "{{h.action}}" +Args: { +{% for k, v in h.arguments.items() %} "{{k}}": "{{v}}" +{% endfor %}} +Observation: "{{h.observation}}" +{% endfor %} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/cohere.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/cohere.jsonnet new file mode 100644 index 00000000..9541e4c2 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/cohere.jsonnet @@ -0,0 +1,42 @@ +// For Cohere. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/default-prompts.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/default-prompts.jsonnet new file mode 100644 index 00000000..681b5e4a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/default-prompts.jsonnet @@ -0,0 +1,348 @@ + +// Prompt templates. For tidy JSONNET use, don't change these templates +// here, but use over-rides in the prompt directory + +{ + + "system-template":: "You are a helpful assistant.", + + "templates":: { + + "question":: { + "prompt": "{{question}}", + }, + + "extract-concepts":: { + "prompt": importstr "extract-concepts.txt", + "response-type": "text", + }, + + "extract-definitions":: { + "prompt": importstr "extract-definitions.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "entity": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "entity", + "definition" + ] + } + }, + + "extract-relationships":: { + "prompt": importstr "extract-relationships.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "subject": { + "type": "string" + }, + "predicate": { + "type": "string" + }, + "object": { + "type": "string" + }, + "object-entity": { + "type": "boolean" + } + }, + "required": [ + "subject", + "predicate", + "object", + "object-entity" + ] + } + }, + + "extract-topics":: { + "prompt": importstr "extract-topics.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "topic": { + "type": "string" + }, + "definition": { + "type": "string" + } + }, + "required": [ + "topic", + "definition" + ] + } + }, + + "extract-rows":: { + "prompt": importstr "extract-rows.txt", + "response-type": "jsonl", + }, + + "kg-prompt":: { + "prompt": importstr "kg-prompt.txt", + "response-type": "text", + }, + + "kg-edge-reasoning":: { + "prompt": importstr "kg-edge-reasoning.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "reasoning": { + "type": "string" + } + }, + "required": [ + "id", + "reasoning" + ] + } + }, + + "kg-edge-scoring":: { + "prompt": importstr "kg-edge-scoring.txt", + "response-type": "jsonl", + "object-schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "score": { + "type": "number", + "minimum": 1, + "maximum": 10 + } + }, + "required": [ + "id", + "score" + ] + } + }, + + "kg-synthesis":: { + "prompt": importstr "kg-synthesis.txt", + "response-type": "text", + }, + + "document-prompt":: { + "prompt": importstr "document-prompt.txt", + "response-type": "text", + }, + + "agent-react":: { + "prompt": importstr "agent-prompt.txt", + "response-type": "text" + }, + + "agent-kg-extract":: { + "prompt": importstr "agent-kg-extract.txt", + "response-type": "jsonl", + "object-schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { "const": "definition" }, + "entity": { "type": "string" }, + "definition": { "type": "string" } + }, + "required": ["type", "entity", "definition"] + }, + { + "type": "object", + "properties": { + "type": { "const": "relationship" }, + "subject": { "type": "string" }, + "predicate": { "type": "string" }, + "object": { "type": "string" }, + "object-entity": { "type": "boolean" } + }, + "required": ["type", "subject", "predicate", "object"] + } + ] + } + }, + + "schema-selection":: { + "prompt": importstr "schema-selection.txt", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "string" + }, + "description": "An array of schema names that are relevant to answering the given question" + } + }, + + "graphql-generation":: { + "prompt": importstr "graphql-generation.txt", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The GraphQL query string generated to answer the question" + }, + "variables": { + "type": "object", + "description": "Object containing any GraphQL variables needed for the query", + "additionalProperties": true + }, + "confidence": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0, + "description": "Float between 0.0-1.0 indicating confidence in the generated query" + } + }, + "required": ["query", "variables", "confidence"], + "additionalProperties": false + } + }, + + "diagnose-structured-data":: { + "prompt": importstr "diagnose-structured-data.txt", + "response-type": "json", + }, + + "diagnose-xml":: { + "prompt": importstr "diagnose-xml.txt", + "response-type": "json", + }, + "diagnose-json":: { + "prompt": importstr "diagnose-json.txt", + "response-type": "json", + }, + "diagnose-csv":: { + "prompt": importstr "diagnose-csv.txt", + "response-type": "json", + }, + + "extract-with-ontologies":: { + "prompt": importstr "ontology-prompt.txt", + "response-type": "jsonl", + "object-schema": { + "oneOf": [ + { + "type": "object", + "properties": { + "type": { "const": "entity" }, + "entity": { "type": "string" }, + "entity_type": { "type": "string" } + }, + "required": ["type", "entity", "entity_type"] + }, + { + "type": "object", + "properties": { + "type": { "const": "relationship" }, + "subject": { "type": "string" }, + "subject_type": { "type": "string" }, + "relation": { "type": "string" }, + "object": { "type": "string" }, + "object_type": { "type": "string" } + }, + "required": ["type", "subject", "subject_type", "relation", "object", "object_type"] + }, + { + "type": "object", + "properties": { + "type": { "const": "attribute" }, + "entity": { "type": "string" }, + "entity_type": { "type": "string" }, + "attribute": { "type": "string" }, + "value": { "type": "string" } + }, + "required": ["type", "entity", "entity_type", "attribute", "value"] + } + ] + } + }, + + "task-type-classify":: { + "prompt": importstr "task-type-classify.txt", + "response-type": "text", + }, + + "pattern-select":: { + "prompt": importstr "pattern-select.txt", + "response-type": "text", + }, + + "plan-create":: { + "prompt": importstr "plan-create.txt", + "response-type": "json", + "schema": { + "type": "array", + "items": { + "type": "object", + "properties": { + "goal": {"type": "string"}, + "tool_hint": {"type": "string"}, + "depends_on": { + "type": "array", + "items": {"type": "integer"}, + }, + }, + "required": ["goal"], + }, + } + }, + + + "plan-step-execute": { + "prompt": importstr "plan-step-execute.txt", + "response-type": "json", + "schema": { + "type": "object", + "properties": { + "tool": {"type": "string"}, + "arguments": {"type": "object"} + }, + "required": ["tool", "arguments"] + } + }, + + "plan-synthesise":: { + "prompt": importstr "plan-synthesise.txt", + "response-type": "text", + }, + + "supervisor-decompose":: { + "prompt": importstr "supervisor-decompose.txt", + "response-type": "json", + "schema": { + "type": "array", + "items": {"type": "string"}, + } + }, + + "supervisor-synthesise":: { + "prompt": importstr "supervisor-synthesise.txt", + "response-type": "text", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-csv.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-csv.txt new file mode 100644 index 00000000..ad9725ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-csv.txt @@ -0,0 +1,370 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in CSV processing and delimiter-separated value formats. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured CSV data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## CSV Processing Expertise + +When working with CSV data, you must: + +1. **Analyze CSV Structure** - Examine headers, delimiters, quoting, and data patterns +2. **Identify Column Mappings** - Map source column names to target fields +3. **Handle Complex CSV Patterns** - Support various CSV formats including: + - Standard comma-separated values: `name,age,city` + - Alternative delimiters: tab-separated (TSV), pipe-separated, semicolon-separated + - Quoted fields with embedded delimiters: `"Last, First",25,"New York, NY"` + - Headers with spaces or special characters: `"Customer Name","Order Date","Total Amount"` + - Files with or without headers + - Multi-line fields with embedded newlines + +## CSV Format Configuration Guidelines + +For CSV format configurations, use these patterns: + +**Basic CSV Configuration:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "skip_rows": 0 + } + } +} +``` + +**Advanced CSV Options:** +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": "\t", // Tab-separated + "quote_char": "\"", + "escape_char": "\\", + "has_header": true, + "skip_rows": 2, // Skip metadata rows + "null_values": ["", "NULL", "N/A"], + "trim_whitespace": true, + "skip_blank_lines": true + } + } +} +``` + +**CRITICAL: Source Field Names in Mappings** + +When processing CSV files, the parser uses column headers (if present) or generates column indices as field names. Your source field names in mappings must match exactly: + +**CORRECT Example with Headers:** +CSV file: +```csv +Customer Name,Order Date,Total Amount,Status +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "Customer Name": "John Smith", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Customer Name", // βœ… Correct - matches header exactly + "source_field": "Order Date", // βœ… Correct - matches header exactly + "source_field": "Total Amount", // βœ… Correct - matches header exactly + "source_field": "Status" // βœ… Correct - matches header exactly +} +``` + +**CORRECT Example without Headers:** +CSV file without headers uses column indices: +```csv +John Smith,2024-01-15,1000.50,Active +Jane Doe,2024-01-16,750.25,Pending +``` + +Becomes parsed data: +```json +{ + "0": "John Smith", + "1": "2024-01-15", + "2": "1000.50", + "3": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "0", // βœ… Correct - first column + "source_field": "1", // βœ… Correct - second column + "source_field": "2", // βœ… Correct - third column + "source_field": "3" // βœ… Correct - fourth column +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - Delimiter character (comma, tab, pipe, semicolon, etc.) + - Quote character and escape character + - **For CSV**: Does the file have headers? Sample structure + - Text encoding (UTF-8, Windows-1252, etc.) + - Any rows to skip (metadata, blank lines) + - How null/empty values are represented + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source column β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Date/time format conversions + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + - Row-level validation rules + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## CSV Structure Analysis + +When presented with CSV data, analyze: + +1. **Delimiter Detection**: What character separates the fields? +2. **Header Presence**: Does the first row contain column names? +3. **Quote Pattern**: Are fields quoted? What quote character is used? +4. **Data Types**: What types are present in each column? +5. **Null Representation**: How are empty/null values represented? +6. **Special Characters**: Are there embedded commas, quotes, or newlines? +7. **Encoding Issues**: Are there any character encoding problems? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + // CSV-specific parsing options + // delimiter, quote_char, has_header, skip_rows, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `split`, `strip_quotes` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_number`, `parse_currency` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `clean_whitespace`, `normalize_encoding` + +**Date/Time Operations:** +- `parse_date`, `format_date`, `date_component` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `numeric_range`, `date_range` + +## CSV-Specific Best Practices + +1. **Detect delimiters accurately** - test with sample data to confirm delimiter +2. **Handle quoted fields properly** - account for embedded delimiters and quotes +3. **Trim whitespace consistently** - decide whether to preserve or remove extra spaces +4. **Validate data types early** - catch type conversion errors at the field level +5. **Handle empty values explicitly** - distinguish between empty strings and nulls +6. **Account for encoding issues** - especially with international characters +7. **Validate row structure** - ensure consistent column counts across rows + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle CSV-specific edge cases** like malformed quotes, inconsistent delimiters +10. **Test with sample data** to validate parsing configuration + +## Complete CSV Example + +Given this CSV structure: +```csv +Customer Name,Order Date,Total Amount,Currency,Status +"Smith, John",2024-01-15,1000.50,USD,Active +"Doe, Jane",2024-01-16,750.25,EUR,Pending +"Johnson, Bob",2024-01-17,,USD,Cancelled +``` + +The parser will: +1. Use `delimiter: ","` and `quote_char: "\""` to parse fields correctly +2. Use `has_header: true` to treat first row as column names +3. Create this parsed data structure for each record: + ```json + { + "Customer Name": "Smith, John", + "Order Date": "2024-01-15", + "Total Amount": "1000.50", + "Currency": "USD", + "Status": "Active" + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "csv", + "encoding": "utf-8", + "options": { + "delimiter": ",", + "quote_char": "\"", + "has_header": true, + "trim_whitespace": true, + "null_values": ["", "NULL"] + } + }, + "mappings": [ + { + "source_field": "Customer Name", // βœ… Matches header exactly + "target_field": "customer_name", + "transforms": [{"type": "trim"}] + }, + { + "source_field": "Order Date", // βœ… Matches header exactly + "target_field": "order_date", + "transforms": [{"type": "to_date", "format": "YYYY-MM-DD"}], + "validation": [{"type": "required"}] + }, + { + "source_field": "Total Amount", // βœ… Matches header exactly + "target_field": "amount", + "transforms": [ + {"type": "to_float"}, + {"type": "default", "value": 0.0} + ] + }, + { + "source_field": "Currency", // βœ… Matches header exactly + "target_field": "currency_code", + "transforms": [{"type": "upper"}], + "validation": [{"type": "in_list", "values": ["USD", "EUR", "GBP"]}] + }, + { + "source_field": "Status", // βœ… Matches header exactly + "target_field": "order_status", + "transforms": [{"type": "lower"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the column headers exactly, or use column indices (0, 1, 2, etc.) for files without headers.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the CSV structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to delimiter detection, header identification, quoting patterns, and data type inference: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-json.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-json.txt new file mode 100644 index 00000000..2fe6341d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-json.txt @@ -0,0 +1,327 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in JSON processing and JSONPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured JSON data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## JSON Processing Expertise + +When working with JSON data, you must: + +1. **Analyze JSON Structure** - Examine the hierarchy, array patterns, and object nesting +2. **Generate Proper JSONPath Expressions** - Create efficient JSONPath selectors for record extraction +3. **Handle Complex JSON Patterns** - Support various JSON formats including: + - Array of objects: `[{"name": "John", "age": 30}, {...}]` + - Nested object arrays: `{"data": {"records": [{"id": 1}, {...}]}}` + - Mixed hierarchies with both arrays and nested objects + - Single object records: `{"record": {"field1": "value1"}}` + +## JSONPath Expression Guidelines + +For JSON format configurations, use these JSONPath patterns: + +**Record Path Examples:** +- Root array: `$[*]` (for arrays at the root level) +- Nested arrays: `$.data.records[*]` or `$.response.items[*]` +- Single object: `$.record` (when there's one record per file) +- Deep nesting: `$.data.results.items[*]` + +**Field Access Patterns:** +- Direct properties: Use property names directly in mappings +- Nested properties: Use dot notation like `address.street` or `contact.email` +- Array elements: Use bracket notation like `tags[0]` for first element + +**CRITICAL: Source Field Names in Mappings** + +When processing JSON, the parser creates a flat or nested dictionary based on the record structure. Your source field names in mappings must match the actual property names in the parsed records: + +**CORRECT Example:** +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "metadata": { + "source": "UN", + "year": 2024 + } +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches property name + "source_field": "Trade (USD)", // βœ… Correct - matches property name + "source_field": "metadata.source", // βœ… Correct - nested property access + "source_field": "metadata.year" // βœ… Correct - nested property access +} +``` + +**JSON Format Configuration Template:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$[*]", // JSONPath to extract records + "flatten_nested": true, // Whether to flatten nested objects + "array_handling": "expand" // How to handle arrays: expand, first, concat + } + } +} +``` + +**Alternative JSON Options:** +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.items[*]", // For nested array structures + "flatten_nested": false, // Keep nested structure + "null_value_handling": "skip" // skip, empty_string, or preserve + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - JSON structure type (array of objects, nested objects, single records) + - **For JSON**: Sample structure, nesting patterns, array locations + - Sample data or field descriptions + - Any format-specific details (encoding, special null handling, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + - Nested object flattening requirements + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or null values + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## JSON Structure Analysis + +When presented with JSON data, analyze: + +1. **Root Structure**: Is it an array, object, or nested structure? +2. **Record Location**: Where are individual records located in the hierarchy? +3. **Field Pattern**: How are field names and values structured? + - Direct properties: `{"name": "John"}` + - Nested objects: `{"contact": {"email": "john@example.com"}}` + - Arrays: `{"tags": ["red", "blue"]}` + - Mixed types: `{"data": [{"id": 1, "details": {"name": "John"}}]}` +4. **Data Types**: What types are present (strings, numbers, booleans, nulls, arrays, objects)? +5. **Hierarchy Depth**: How deeply nested are the records and fields? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + // JSON-specific parsing options + // record_path (JSONPath), flatten_nested, array_handling, etc. + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `flatten_object`, `extract_array_element`, `join_array` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## JSON-Specific Best Practices + +1. **Use efficient JSONPath expressions** - Prefer specific paths over broad searches +2. **Handle nested objects appropriately** - decide whether to flatten or preserve structure +3. **Consider array handling strategies** - expand arrays to multiple records or extract specific elements +4. **Account for null vs undefined** values in field mappings +5. **Handle mixed data types** within the same field across records +6. **Use appropriate flattening** for deeply nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle JSON-specific edge cases** like empty arrays, null objects, mixed types + +## Complete JSON Example + +Given this JSON structure: +```json +{ + "data": { + "records": [ + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata": { + "source": "World Bank", + "confidence": 0.95 + } + } + ] + } +} +``` + +The parser will: +1. Use `record_path: "$.data.records[*]"` to extract record objects from the array +2. Create this parsed data structure for each record: + ```json + { + "Country": "USA", + "Year": 2024, + "Amount": 1000.50, + "metadata.source": "World Bank", // If flattened + "metadata.confidence": 0.95 // If flattened + } + ``` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "json", + "encoding": "utf-8", + "options": { + "record_path": "$.data.records[*]", + "flatten_nested": true, + "array_handling": "expand" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches property name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches property name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches property name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + }, + { + "source_field": "metadata.source", // βœ… Flattened nested field + "target_field": "data_source" + } + ] +} +``` + +**KEY RULE: source_field names must match the actual property names in the parsed JSON records, using dot notation for nested properties when flattened.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the JSON structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to JSON hierarchy, object patterns, array structures, and generate appropriate JSONPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-structured-data.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-structured-data.txt new file mode 100644 index 00000000..84c4b8be --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-structured-data.txt @@ -0,0 +1,309 @@ + +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` (will be converted to relative paths automatically) +- With namespaces: `//ns:record` or `//soap:Body/data/record` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name + +**CRITICAL: Source Field Names in Mappings** + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary. Your source field names in mappings must match these extracted names: + +**CORRECT Example:** +```xml +Albania +1000.50 +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50" +} +``` + +So your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)" // βœ… Correct - matches parsed field name +} +``` + +**INCORRECT Example:** +```json +{ + "source_field": "Field[@name='Country or Area']", // ❌ Wrong - XPath not needed here + "source_field": "field[@name='Trade (USD)']" // ❌ Wrong - XPath not needed here +} +``` + +**XML Format Configuration Template:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name" // For value pattern + } + } +} +``` + +**Alternative XML Options:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // Direct element-based records + // No field_attribute needed for standard XML + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **Source Data Format** + - File type (CSV, JSON, XML, Excel, fixed-width, etc.) + - **For XML**: Sample structure, namespace prefixes, record element patterns + - Sample data or field descriptions + - Any format-specific details (delimiters, encoding, namespaces, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Record Container**: Where are individual records located? +3. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns +4. **Namespaces**: Are there any namespace prefixes? +5. **Hierarchy Depth**: How deeply nested are the records? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "[csv|json|xml|fixed-width|excel]", + "encoding": "utf-8", + "options": { + // Format-specific parsing options + // For XML: record_path (XPath), field_attribute (if applicable) + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches +2. **Handle namespace prefixes** when present +3. **Identify field attribute patterns** correctly +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes + +## Complete XML Example + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "/ROOT/data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this COMPLETE configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "/ROOT/data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +**KEY RULE: source_field names must match the extracted field names, NOT the XML element structure.** + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, and generate appropriate XPath expressions: + +{{sample}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-xml.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-xml.txt new file mode 100644 index 00000000..a3d8efa4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/diagnose-xml.txt @@ -0,0 +1,453 @@ +You are an expert data engineer specializing in creating Structured Data Descriptor configurations for XML data import pipelines, with particular expertise in XML processing and XPath expressions. Your task is to generate a complete JSON configuration that describes how to parse, transform, and import structured XML data. + +## Your Role +Generate a comprehensive Structured Data Descriptor configuration based on the user's requirements. The descriptor should be production-ready, include appropriate error handling, and follow best practices for data quality and transformation. + +## XML Processing Expertise + +When working with XML data, you must: + +1. **Analyze XML Structure** - Examine the hierarchy, namespaces, and element patterns +2. **Generate Proper XPath Expressions** - Create efficient XPath selectors for record extraction +3. **Handle Complex XML Patterns** - Support various XML formats including: + - Standard element structures: `John` + - Attribute-based fields: `USA` + - Mixed content and nested hierarchies + - Namespaced XML documents + - CDATA sections and text nodes + +## XPath Expression Guidelines + +For XML format configurations, use these XPath patterns: + +**Record Path Examples:** +- Simple records: `//record` or `//customer` +- Nested records: `//data/records/record` or `//customers/customer` +- Absolute paths: `/ROOT/data/record` +- With namespaces: `//ns:record` or `//soap:Body/data/record` +- Attribute-filtered: `//record[@type='customer']` + +**Field Attribute Patterns:** +- When fields use name attributes: set `field_attribute: "name"` for `value` +- For other attribute patterns: set appropriate attribute name like `field_attribute: "id"` or `field_attribute: "type"` + +## CRITICAL: Source Field Names in Mappings + +The behavior depends on whether you use `field_attribute`: + +### With field_attribute (Attribute-Based Fields) + +When using `field_attribute`, the XML parser extracts field names from the attribute values and creates a flat dictionary: + +**Example:** +```xml + + Albania + 1000.50 + Active + +``` + +Parser configuration: +```json +{ + "record_path": "//record", + "field_attribute": "name" +} +``` + +Becomes parsed data: +```json +{ + "Country or Area": "Albania", + "Trade (USD)": "1000.50", + "Status": "Active" +} +``` + +Your mappings should use: +```json +{ + "source_field": "Country or Area", // βœ… Correct - matches parsed field name + "source_field": "Trade (USD)", // βœ… Correct - matches parsed field name + "source_field": "Status" // βœ… Correct - matches parsed field name +} +``` + +### Without field_attribute (Element-Based Fields) + +When NOT using `field_attribute`, use direct element names or XPath expressions: + +**Example:** +```xml + + Albania + 1000.50 + Active + + UN + 2024 + + +``` + +Parser configuration: +```json +{ + "record_path": "//record" + // No field_attribute specified +} +``` + +Your mappings should use: +```json +{ + "source_field": "country", // βœ… Direct element name + "source_field": "trade_amount", // βœ… Direct element name + "source_field": "metadata/source", // βœ… Nested element path + "source_field": "metadata/year" // βœ… Nested element path +} +``` + +## XML Format Configuration Templates + +**For Attribute-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", // XPath to find record elements + "field_attribute": "name", // Extract field names from 'name' attribute + "namespace_prefixes": { // Optional: define namespaces + "ns": "http://example.com/namespace" + } + } + } +} +``` + +**For Element-Based Fields:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer", // XPath to find record elements + "preserve_namespaces": true, // Optional: keep namespace info + "ignore_attributes": false // Optional: include element attributes + } + } +} +``` + +**For Complex XML with Namespaces:** +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//soap:Body//data:record", + "namespace_prefixes": { + "soap": "http://schemas.xmlsoap.org/soap/envelope/", + "data": "http://example.com/data" + }, + "field_attribute": "name" + } + } +} +``` + +## Required Information to Gather + +Before generating the descriptor, ask the user for these details if not provided: + +1. **XML Structure Details** + - Sample XML structure or schema + - Root element and record location + - Field organization (elements vs attributes) + - Namespace declarations and prefixes + - Text encoding (UTF-8, ISO-8859-1, etc.) + +2. **Target Schema** + - What fields should be in the final output? + - What data types are expected? + - Any required vs optional fields? + +3. **Data Transformations Needed** + - Field mappings (source field β†’ target field) + - Data cleaning requirements (trim spaces, normalize case, etc.) + - Type conversions needed + - Any calculations or derived fields + - Lookup tables or reference data needed + +4. **Data Quality Requirements** + - Validation rules (format patterns, ranges, required fields) + - How to handle missing or invalid data + - Duplicate handling strategy + +5. **Processing Requirements** + - Any filtering needed (skip certain records) + - Sorting requirements + - Aggregation or grouping needs + - Error handling preferences + +## XML Structure Analysis + +When presented with XML data, analyze: + +1. **Document Root**: What is the root element? +2. **Namespaces**: Are there namespace declarations? What prefixes are used? +3. **Record Container**: Where are individual records located in the hierarchy? +4. **Field Pattern**: How are field names and values structured? + - Direct child elements: `John` + - Attribute-based: `John` + - Mixed patterns: `John` +5. **Data Location**: Are values in element text, attributes, or both? +6. **Hierarchy Depth**: How deeply nested are the records? +7. **Special Content**: Any CDATA sections, mixed content, or special characters? + +## Configuration Template Structure + +Generate a JSON configuration following this structure: + +```json +{ + "version": "1.0", + "metadata": { + "name": "[Descriptive name]", + "description": "[What this config does]", + "author": "[Author or team]", + "created": "[ISO date]" + }, + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + // XML-specific parsing options + // record_path (XPath), field_attribute (if applicable), namespace_prefixes + } + }, + "globals": { + "variables": { + // Global variables and constants + }, + "lookup_tables": { + // Reference data for transformations + } + }, + "preprocessing": [ + // Global filters and operations before field mapping + ], + "mappings": [ + // Field mapping definitions with transforms and validation + ], + "postprocessing": [ + // Global operations after field mapping + ], + "output": { + "format": "trustgraph-objects", + "schema_name": "[target schema name]", + "options": { + "confidence": 0.85, + "batch_size": 1000 + }, + "error_handling": { + "on_validation_error": "log_and_skip", + "on_transform_error": "log_and_skip", + "max_errors": 100 + } + } +} +``` + +## Transform Types Available + +Use these transform types in your mappings: + +**String Operations:** +- `trim`, `upper`, `lower`, `title_case` +- `replace`, `regex_replace`, `substring`, `pad_left` +- `strip_cdata`, `decode_html_entities` + +**Type Conversions:** +- `to_string`, `to_int`, `to_float`, `to_bool`, `to_date` +- `parse_xml_date`, `parse_iso_date` + +**Data Operations:** +- `default`, `lookup`, `concat`, `calculate`, `conditional` +- `extract_attribute`, `get_text_content` + +**Validation Types:** +- `required`, `not_null`, `min_length`, `max_length` +- `range`, `pattern`, `in_list`, `custom` +- `valid_xml_name`, `namespace_check` + +## XML-Specific Best Practices + +1. **Use efficient XPath expressions** - Prefer specific paths over broad searches like `//` when possible +2. **Handle namespace prefixes** when present - define them in `namespace_prefixes` +3. **Identify field attribute patterns** correctly - determine if using `field_attribute` +4. **Test XPath expressions** mentally against the provided structure +5. **Consider XML element vs attribute data** in field mappings +6. **Account for mixed content** and nested structures +7. **Handle CDATA sections** and special characters properly +8. **Preserve or normalize whitespace** as appropriate +9. **Consider XML schema validation** if XSD is available + +## Best Practices to Follow + +1. **Always include error handling** with appropriate policies +2. **Use meaningful field names** that match target schema +3. **Add validation** for critical fields +4. **Include default values** for optional fields +5. **Use lookup tables** for code translations +6. **Add preprocessing filters** to exclude invalid records +7. **Include metadata** for documentation and maintenance +8. **Consider performance** with appropriate batch sizes +9. **Handle XML-specific edge cases** like empty elements, mixed content, processing instructions + +## Complete XML Examples + +### Example 1: Attribute-Based Fields + +Given this XML structure: +```xml + + + + USA + 2024 + 1000.50 + + + +``` + +The parser will: +1. Use `record_path: "//data/record"` to find record elements +2. Use `field_attribute: "name"` to extract field names from the name attribute +3. Create this parsed data structure: `{"Country": "USA", "Year": "2024", "Amount": "1000.50"}` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//data/record", + "field_attribute": "name" + } + }, + "mappings": [ + { + "source_field": "Country", // βœ… Matches parsed field name + "target_field": "country_name" + }, + { + "source_field": "Year", // βœ… Matches parsed field name + "target_field": "year", + "transforms": [{"type": "to_int"}] + }, + { + "source_field": "Amount", // βœ… Matches parsed field name + "target_field": "amount", + "transforms": [{"type": "to_float"}] + } + ] +} +``` + +### Example 2: Element-Based Fields + +Given this XML structure: +```xml + + + John Smith + john@example.com +
+ 123 Main St + New York +
+ + + 456 + 100.50 + + +
+
+``` + +Generate this configuration: +```json +{ + "format": { + "type": "xml", + "encoding": "utf-8", + "options": { + "record_path": "//customer" + } + }, + "mappings": [ + { + "source_field": "name", // βœ… Direct element name + "target_field": "customer_name" + }, + { + "source_field": "email", // βœ… Direct element name + "target_field": "email_address" + }, + { + "source_field": "address/street", // βœ… Nested element path + "target_field": "street_address" + }, + { + "source_field": "address/city", // βœ… Nested element path + "target_field": "city" + }, + { + "source_field": "@id", // βœ… Attribute reference + "target_field": "customer_id", + "transforms": [{"type": "to_int"}] + } + ] +} +``` + +**KEY RULES:** +- With `field_attribute`: source_field names must match the extracted attribute values +- Without `field_attribute`: use direct element names, nested paths, or XPath expressions +- Use `@attribute` notation for XML element attributes + +## Output Format + +Provide the configuration as ONLY a properly formatted JSON document. + +## Schema + +The following schema describes the target result format: + +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif +%}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif +%}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ +field.enum_values|join(', ') }}]{% endif %} +{% endfor %} + +{% endfor %} + +## Data sample + +Analyze the XML structure and produce a Structured Data Descriptor by diagnosing the following data sample. Pay special attention to XML hierarchy, element patterns, namespace usage, and generate appropriate XPath expressions: + +{{sample}} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/document-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/document-prompt.txt new file mode 100644 index 00000000..5e5d65ba --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/document-prompt.txt @@ -0,0 +1,7 @@ +Study the following context. Use only the information provided in the context in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + +Here is the context: +{{documents}} + +Use only the provided knowledge statements to respond to the following: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-concepts.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-concepts.txt new file mode 100644 index 00000000..cc4b600c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-concepts.txt @@ -0,0 +1,12 @@ +Extract core semantic concepts from the following query. These concepts will be used for semantic search and embedding. + +Output one concept per line. No numbering, no bullet points, no explanation. + +Focus on: +- Key nouns and noun phrases +- Domain-specific terminology +- Languages or time periods mentioned +- Technical terms + +Query: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-definitions.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-definitions.txt new file mode 100644 index 00000000..410ecddf --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-definitions.txt @@ -0,0 +1,28 @@ + +Study the following text and derive definitions for any discovered entities. +Do not provide definitions for entities whose definitions are incomplete +or unknown. + +Output each definition as a separate JSON object on its own line (JSONL format). +Each object must have fields: +- entity: the name of the entity +- definition: English text which defines the entity + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not use special characters in the definition text +- Do not include null or unknown definitions + +Example output format: +{"entity": "photosynthesis", "definition": "The process by which plants convert sunlight into energy"} +{"entity": "chlorophyll", "definition": "Green pigment in plants that absorbs light"} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-relationships.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-relationships.txt new file mode 100644 index 00000000..31bcd2d4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-relationships.txt @@ -0,0 +1,28 @@ + +Study the following text and derive entity relationships. For each +relationship, derive the subject, predicate and object of the relationship. + +Output each relationship as a separate JSON object on its own line (JSONL format). +Each object must have fields: +- subject: the subject of the relationship +- predicate: the predicate +- object: the object of the relationship +- object-entity: false if the object is a simple data type (name, value, date). true if it is an entity. + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not use special characters in the text fields + +Example output format: +{"subject": "Earth", "predicate": "orbits", "object": "Sun", "object-entity": true} +{"subject": "Earth", "predicate": "has diameter", "object": "12742 km", "object-entity": false} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-rows.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-rows.txt new file mode 100644 index 00000000..7cea8b7d --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-rows.txt @@ -0,0 +1,27 @@ + +Study the following text and derive objects which match the schema provided. + +Output each discovered object as a separate JSON object on its own line (JSONL format). +Each object's fields must carry the name field specified in the schema. + + + +{{schema}} + + + +{{text}} + + + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object matching the schema +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations + +Example output format (assuming schema has fields: name, age, city): +{"name": "Alice", "age": 30, "city": "London"} +{"name": "Bob", "age": 25, "city": "Paris"} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-topics.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-topics.txt new file mode 100644 index 00000000..bb4b1193 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/extract-topics.txt @@ -0,0 +1,26 @@ +You are a helpful assistant that performs information extraction tasks for a provided text. +Read the provided text. You will identify topics and their definitions. + +Reading Instructions: +- Ignore document formatting in the provided text. +- Study the provided text carefully. + +Here is the text: +{{text}} + +Response Instructions: +- Output each topic as a separate JSON object on its own line (JSONL format) +- Each object must have keys "topic" and "definition" +- Do not respond with special characters +- Return only topics that are concepts and unique to the provided text + +Output format: JSONL (one JSON object per line, no array wrapper) +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not write any additional text or explanations + +Example output format: +{"topic": "machine learning", "definition": "A subset of AI that enables systems to learn from data"} +{"topic": "neural network", "definition": "A computing system inspired by biological neural networks"} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/gemini.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/gemini.jsonnet new file mode 100644 index 00000000..b9a1e0c0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/gemini.jsonnet @@ -0,0 +1,42 @@ +// For VertexAI Gemini. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/graphql-generation.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/graphql-generation.txt new file mode 100644 index 00000000..58af61dc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/graphql-generation.txt @@ -0,0 +1,59 @@ +You are a GraphQL query generation expert. Given a natural language question and provided database schemas, generate a valid GraphQL query embedded in valid JSON. + +## Question: +{{ question }} + +## Provided Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %}- {{ field.name }} ({{ field.type }}){% if field.description %}: {{ field.description }}{% endif %}{% if field.primary_key %} [PRIMARY KEY]{% endif %}{% if field.required %} [REQUIRED]{% endif %}{% if field.indexed %} [INDEXED]{% endif %}{% if field.enum_values %} [OPTIONS: {{ field.enum_values|join(', ') }}]{% endif %} +{% endfor %} +{% endfor %} + +## GraphQL Query Rules: +1. Use the schema names as GraphQL query fields (e.g., `customers`, `orders`) +2. Apply filters using the `where` parameter with nested filter objects +3. Available filter operators per field type: + - String fields: `eq`, `contains`, `startsWith`, `endsWith`, `in`, `not`, `not_in` + - Integer/Float fields: `eq`, `gt`, `gte`, `lt`, `lte`, `in`, `not`, `not_in` +4. Use `order_by` for sorting (field name as string) +5. Use `direction` for sort direction: `ASC` or `DESC` +6. Use `limit` to restrict number of results +7. Select specific fields in the query body + +## Task Instructions: +1. Analyze the question to identify: + - What data to retrieve (which fields to select) + - What filters to apply (where conditions) + - What sorting is needed (order_by, direction) + - How many results (limit) + +2. Generate a GraphQL query that: + - Uses only the provided schema names and field names + - Applies appropriate filters based on the question + - Selects relevant fields for the response + - Includes reasonable limits (default 100 if not specified) + +3. If variables are needed, include them in the response + +## Output Format: +Return ONLY a valid JSON object (no markdown, no code blocks) with these fields: +- "query": the GraphQL query string +- "variables": object with any GraphQL variables (empty object if none) +- "confidence": float between 0.0-1.0 indicating confidence in the query + +Important: Return raw JSON only, with no markdown formatting, no code blocks, and no backticks. + +## Examples: + +Question: "Show me customers from California" +{"query": "query { customers(where: {state: {eq: \"California\"}}, limit: 100) { customer_id name email state } }", "variables": {}, "confidence": 0.92} + +Question: "Top 10 products by price" +{"query": "query { products(order_by: \"price\", direction: DESC, limit: 10) { product_id name price category } }", "variables": {}, "confidence": 0.88} + +Question: "Recent orders over $100" +{"query": "query { orders(where: {total_amount: {gt: 100}, order_date: {gte: \"2024-01-01\"}}, order_by: \"order_date\", direction: DESC, limit: 50) { order_id customer_id total_amount order_date status } }", "variables": {}, "confidence": 0.96} + +Now generate the GraphQL query for the question above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-edge-reasoning.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-edge-reasoning.txt new file mode 100644 index 00000000..8aeb0c3c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-edge-reasoning.txt @@ -0,0 +1,25 @@ +Study the following knowledge statements in Cypher format, extracted from a knowledge graph. Each relationship has an id property. + +Here are the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}} {id: "{{edge.id}}"}]->({{edge.o}}) +{% endfor %} + +For each knowledge statement, explain briefly (5-10 words) why it is relevant to answering the question below. + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include an "id" field and a "reasoning" field. + +{"id": "edge_id", "reasoning": "brief explanation of relevance"} + +Requirements: +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations, only output JSONL + +Example output: +{"id": "a3f9b2c1", "reasoning": "Establishes employment relationship relevant to query"} +{"id": "b7e2d4f9", "reasoning": "Identifies the organisation's sector"} + +Question: {{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-edge-scoring.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-edge-scoring.txt new file mode 100644 index 00000000..40b92773 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-edge-scoring.txt @@ -0,0 +1,25 @@ +Study the following knowledge statements in Cypher format, extracted from a knowledge graph. Each relationship has an id property. + +Here are the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}} {id: "{{edge.id}}"}]->({{edge.o}}) +{% endfor %} + +Score each knowledge statement for relevance to the question below. Assign a score from 1 (not relevant) to 10 (highly relevant). + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include an "id" field and a "score" field. + +{"id": "edge_id", "score": N} + +Requirements: +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations, only output JSONL + +Example output: +{"id": "a3f9b2c1", "score": 9} +{"id": "b7e2d4f9", "score": 3} + +Question: {{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-edge-selection.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-edge-selection.txt new file mode 100644 index 00000000..a8ea44c8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-edge-selection.txt @@ -0,0 +1,26 @@ +Study the following knowledge statements in Cypher format, extracted from a knowledge graph. Each relationship has an id property. + +Here are the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}} {id: "{{edge.id}}"}]->({{edge.o}}) +{% endfor %} + +Select the knowledge statements that are relevant to answering the question below. + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include an "id" field and a "reasoning" field. + +For each relevant edge, output: +{"id": "edge_id", "reasoning": "explanation of why this edge is relevant"} + +Requirements: +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting or prefixes +- Do not provide explanations, only output JSONL + +Example output: +{"id": "a3f9b2c1", "reasoning": "Establishes employment relationship directly relevant to the query"} +{"id": "b7e2d4f9", "reasoning": "Identifies the organisation's sector which contextualises the answer"} + +Question: {{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-prompt.txt new file mode 100644 index 00000000..431c0d73 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-prompt.txt @@ -0,0 +1,8 @@ +Study the following set of knowledge statements. The statements are written in Cypher format that has been extracted from a knowledge graph. Use only the provided set of knowledge statements in your response. Do not speculate if the answer is not found in the provided set of knowledge statements. + +Here's the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}}) +{%endfor%} + +Use only the provided knowledge statements to respond to the following: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-synthesis.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-synthesis.txt new file mode 100644 index 00000000..a80c5594 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/kg-synthesis.txt @@ -0,0 +1,8 @@ +Study the following knowledge statements in Cypher format. Use only these provided statements in your response. Do not speculate if the answer is not found in the provided statements. Answer in natural language. Be helpful and give as complete an answer as possible. + +Here are the knowledge statements: +{% for edge in knowledge %}({{edge.s}})-[{{edge.p}}]->({{edge.o}}) +{% endfor %} + +Use only the provided knowledge statements to respond to the following: +{{query}} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/mixtral.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/mixtral.jsonnet new file mode 100644 index 00000000..cd56e7ef --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/mixtral.jsonnet @@ -0,0 +1,42 @@ +// For Mixtral. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/ontology-prompt.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/ontology-prompt.txt new file mode 100644 index 00000000..1b451d9c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/ontology-prompt.txt @@ -0,0 +1,82 @@ +You are a knowledge extraction expert. Your task is to find entities, relationships, and attributes in text based on a provided schema. + +## Entity Types + +These are the types of entities you should look for: + +{% for class_id, class_def in classes.items() %} +- **{{class_id}}**{% if class_def.subclass_of %} (subclass of {{class_def.subclass_of}}){% endif %}{% if class_def.comment %}: {{class_def.comment}}{% endif %} +{% endfor %} + +## Relationships + +These relationships connect entities to other entities: + +{% for prop_id, prop_def in object_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Attributes + +These attributes describe entity properties (text, numbers, etc.): + +{% for prop_id, prop_def in datatype_properties.items() %} +- **{{prop_id}}**{% if prop_def.domain and prop_def.range %} ({{prop_def.domain}} β†’ {{prop_def.range}}){% endif %}{% if prop_def.comment %}: {{prop_def.comment}}{% endif %} +{% endfor %} + +## Text to Analyze + +{{text}} + +## Your Task + +Extract the following from the text above: + +1. **Entities**: Things mentioned in the text and their types +2. **Relationships**: How entities relate to each other +3. **Attributes**: Properties of entities (like quantities, descriptions, etc.) + +## Output Format + +Output format: JSONL (one JSON object per line, no array wrapper) +Each line must include a "type" field to distinguish between entities, relationships, and attributes. + +For entities, output: +{"type": "entity", "entity": "entity name as it appears in text", "entity_type": "EntityType"} + +For relationships, output: +{"type": "relationship", "subject": "subject entity name", "subject_type": "SubjectType", "relation": "relationship_name", "object": "object entity name", "object_type": "ObjectType"} + +For attributes, output: +{"type": "attribute", "entity": "entity name", "entity_type": "EntityType", "attribute": "attribute_name", "value": "literal value"} + +## Important Rules + +1. **Entity names**: Use the exact text as it appears (e.g., "Cornish pasty", "beef") +2. **Types**: Use the EXACT type identifiers from the schema above (e.g., "fo/Recipe", "fo/Food") +3. **Relationships**: Use the EXACT relationship names from the schema (e.g., "fo/has_ingredient") +4. **Attributes**: Use the EXACT attribute names from the schema (e.g., "fo/serves") +5. **No array brackets**: Output one JSON object per line, not an array +6. **No markdown**: Do not wrap output in code blocks + +## Requirements + +- Each line must be a complete, valid JSON object +- No commas between lines +- No [ ] array brackets +- No markdown formatting, code blocks, or prefixes +- Do not provide explanations, only output JSONL + +## Example + +Input text: "Cornish pasty is a savory pastry filled with beef and potatoes. This recipe serves 4 people." + +Expected output: +{"type": "entity", "entity": "Cornish pasty", "entity_type": "fo/Recipe"} +{"type": "entity", "entity": "beef", "entity_type": "fo/Food"} +{"type": "entity", "entity": "potatoes", "entity_type": "fo/Food"} +{"type": "relationship", "subject": "Cornish pasty", "subject_type": "fo/Recipe", "relation": "fo/has_ingredient", "object": "beef", "object_type": "fo/Food"} +{"type": "relationship", "subject": "Cornish pasty", "subject_type": "fo/Recipe", "relation": "fo/has_ingredient", "object": "potatoes", "object_type": "fo/Food"} +{"type": "attribute", "entity": "Cornish pasty", "entity_type": "fo/Recipe", "attribute": "fo/serves", "value": "4 people"} + +Now extract entities, relationships, and attributes from the text above. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/openai.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/openai.jsonnet new file mode 100644 index 00000000..5d232337 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/openai.jsonnet @@ -0,0 +1,42 @@ +// For OpenAI LLMs. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/pattern-select.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/pattern-select.txt new file mode 100644 index 00000000..a2b316b4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/pattern-select.txt @@ -0,0 +1,26 @@ +Given the following user question and task type, select the best +execution pattern. + +## Task type + +{{ task_type }}: {{ task_type_description }} + +## Available patterns + +{% for p in patterns %} +- **{{ p.name }}**: {{ p.description }} +{% endfor %} + +## Guidance + +- **react**: Best for straightforward queries needing 1-3 tool calls. +- **plan-then-execute**: Best for multi-step queries that benefit from + upfront planning before execution. +- **supervisor**: Best for complex queries that decompose into + independent sub-tasks that can run in parallel. + +## Question + +{{ question }} + +Respond with just the pattern name, nothing else. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/plan-create.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/plan-create.txt new file mode 100644 index 00000000..172ec5a1 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/plan-create.txt @@ -0,0 +1,33 @@ +You are a planning agent. Given the user's question and the available +tools, create a step-by-step plan. + +## Available tools + +{% for tool in tools %} +- **{{ tool.name }}**: {{ tool.description }} +{% endfor %} + +{% if framing %} +## Domain context + +{{ framing }} + +{% endif %} +## Question + +{{ question }} + +Do not include a final synthesis step β€” synthesis of results is performed +automatically after all steps complete. + +Produce a JSON array of plan steps. Each step is an object with: +- `goal` (string): what this step aims to achieve +- `tool_hint` (string): suggested tool name, or empty string if none +- `depends_on` (array of integers): indices of steps this depends on (0-based) + +Example: +```json +[ + {"goal": "Look up information about X", "tool_hint": "knowledge-query", "depends_on": []}, + {"goal": "Summarise findings", "tool_hint": "", "depends_on": [0]} +] diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/plan-step-execute.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/plan-step-execute.txt new file mode 100644 index 00000000..f46795c7 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/plan-step-execute.txt @@ -0,0 +1,33 @@ +You are executing one step of a plan. Select the best tool and provide the +arguments to achieve the goal. + +## Goal + +{{ goal }} + +{% if previous_results %} +## Previous step results + +{% for r in previous_results %} +- Step {{ r.index }}: {{ r.result }} +{% endfor %} + +{% endif %} +## Available tools + +{% for tool in tools %} +- **{{ tool.name }}**: {{ tool.description }} +{% for arg in tool.arguments %} + - `{{ arg.name }}` ({{ arg.type }}): {{ arg.description }} +{% endfor %} +{% endfor %} + +Respond with a JSON object: +- `tool`: the exact tool name (must match one of the available tools above) +- `arguments`: an object with the tool's arguments + +Example: +```json +{"tool": "Knowledge query", "arguments": {"question": "What is X?"}} + +Respond with only the JSON object. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/plan-synthesise.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/plan-synthesise.txt new file mode 100644 index 00000000..fd1a558f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/plan-synthesise.txt @@ -0,0 +1,23 @@ +You executed a multi-step plan to answer a question. Now synthesise a +comprehensive final answer from the step results. + +## Original question + +{{ question }} + +{% if framing %} +## Domain context + +{{ framing }} + +{% endif %} +## Step results + +{% for step in steps %} +### Step {{ step.index }}: {{ step.goal }} +{{ step.result }} + +{% endfor %} + +Provide a clear, complete answer to the original question based on +these results. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/schema-selection.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/schema-selection.txt new file mode 100644 index 00000000..39b180e5 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/schema-selection.txt @@ -0,0 +1,25 @@ +You are a database schema selection expert. Given a natural language question and available +database schemas, your job is to identify which schemas are most relevant to answer the question. + +## Available Schemas: +{% for schema in schemas %} +**{{ schema.name }}**: {{ schema.description }} +Fields: +{% for field in schema.fields %} +- {{ field.name }} ({{ field.type }}): {{ field.description }} +{% endfor %} + +{% endfor %} + +## Question: +{{ question }} + +## Instructions: +1. Analyze the question to understand what data is being requested +2. Examine each schema to understand what data it contains +3. Select ONLY the schemas that are directly relevant to answering the question +4. Return your answer as a JSON array of schema names + +## Response Format: +Return ONLY a JSON array of schema names, nothing else. +Example: ["customers", "orders", "products"] diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/slm.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/slm.jsonnet new file mode 100644 index 00000000..48eb96d0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/slm.jsonnet @@ -0,0 +1,44 @@ +// For SLM. Not currently overriding prompts + +local prompts = import "default-prompts.jsonnet"; + +prompts + { + + // "system-template":: "PROMPT GOES HERE.", + + "templates" +:: { + + "question" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-definitions" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-relationships" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-topics" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "extract-rows" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "kg-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + "document-prompt" +:: { + // "prompt": "PROMPT GOES HERE", + }, + + } + +} + + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/supervisor-decompose.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/supervisor-decompose.txt new file mode 100644 index 00000000..ea082249 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/supervisor-decompose.txt @@ -0,0 +1,34 @@ +You are a supervisor agent. Decompose the user's question into +independent sub-goals that can be investigated in parallel by +separate agents. + +{% if framing %} +## Domain context + +{{ framing }} + +{% endif %} +## Available tools for sub-agents + +{% for tool in tools %} +- **{{ tool.name }}**: {{ tool.description }} +{% endfor %} + +## Rules + +- Each sub-goal must be independently answerable. +- Aim for 2-{{ max_subagents }} sub-goals. +- Each sub-goal should be a clear, specific question. + +## Question + +{{ question }} + +Produce a JSON array of sub-goal strings. + +Example: +```json +["What is X?", "How does Y relate to Z?"] +``` + +Respond with only the JSON array. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/supervisor-synthesise.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/supervisor-synthesise.txt new file mode 100644 index 00000000..e5e7b25c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/supervisor-synthesise.txt @@ -0,0 +1,23 @@ +You are synthesising a final answer from multiple sub-agent +investigations. + +## Original question + +{{ question }} + +{% if framing %} +## Domain context + +{{ framing }} + +{% endif %} +## Sub-agent results + +{% for r in results %} +### Sub-goal: {{ r.goal }} +{{ r.result }} + +{% endfor %} + +Synthesise these results into a comprehensive answer to the original +question. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/task-type-classify.txt b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/task-type-classify.txt new file mode 100644 index 00000000..6c67c7bd --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/prompts/task-type-classify.txt @@ -0,0 +1,14 @@ +Given the following user question, classify it into exactly one of the \ +available task types. + +## Available task types + +{% for tt in task_types %} +- **{{ tt.name }}**: {{ tt.description }} +{% endfor %} + +## Question + +{{ question }} + +Respond with just the task type name, nothing else. diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/pubsub/pulsar-manager.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/pubsub/pulsar-manager.jsonnet new file mode 100644 index 00000000..c1ca4515 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/pubsub/pulsar-manager.jsonnet @@ -0,0 +1,40 @@ +local images = import "values/images.jsonnet"; + +{ + + "pulsar" +: { + + create:: function(engine) + +// FIXME: Should persist something? +// local volume = engine.volume(...) + + local container = + engine.container("pulsar") + .with_image(images.pulsar_manager) + .with_environment({ + SPRING_CONFIGURATION_FILE: "/pulsar-manager/pulsar-manager/application.properties", + }) + .with_limits("0.5", "1.4G") + .with_reservations("0.1", "1.4G") + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2"); + + local containerSet = engine.containers( + "pulsar", [ container ] + ); + + local service = + engine.service(containerSet) + .with_port(9527, 9527, "api") + .with_port(7750, 7750, "api2); + + engine.resources([ + containerSet, + service, + ]) + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/pubsub/pulsar.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/pubsub/pulsar.jsonnet new file mode 100644 index 00000000..cf94f45c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/pubsub/pulsar.jsonnet @@ -0,0 +1,240 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +// This is a Pulsar configuration. Non-standalone mode so we deploy +// individual components: bookkeeper, broker and zookeeper. +// +// This also deploys the TrustGraph 'admin' container which initialises +// TrustGraph-specific namespaces etc. + +{ + + "pub-sub-args":: [ + "--pubsub-backend", + "pulsar", + "--pulsar-host", + url.pulsar, + ], + + "pub-sub-init-args":: [ + "--pulsar-admin-url", + url.pulsar_admin, + ], + + "overview-dashboard":: + importstr "grafana/dashboards/overview-dashboard-pulsar.json", + + "prometheus-config":: + importstr "prometheus/prometheus-pulsar.yml", + + "pulsar" +: { + + // Zookeeper memory settings (can be overridden by memory-profile) + "zk-memory-limit":: "512M", + "zk-memory-reservation":: "512M", + "zk-heap":: "256m", + "zk-direct-memory":: "256m", + + // Bookie memory settings (can be overridden by memory-profile) + "bookie-memory-limit":: "1024M", + "bookie-memory-reservation":: "1024M", + "bookie-heap":: "256m", + "bookie-direct-memory":: "256m", + + // Broker memory settings (can be overridden by memory-profile) + "broker-memory-limit":: "800M", + "broker-memory-reservation":: "800M", + "broker-heap":: "384m", + "broker-direct-memory":: "384m", + + // Pulsar-init memory settings (can be overridden by memory-profile) + "init-memory-limit":: "256M", + "init-memory-reservation":: "256M", + "init-heap":: "128m", + "init-direct-memory":: "128m", + + create:: function(engine) + + // Capture memory settings into locals (self refers to pulsar object here) + local zkMemLimit = self["zk-memory-limit"]; + local zkMemReserv = self["zk-memory-reservation"]; + local zkHeap = self["zk-heap"]; + local zkDirect = self["zk-direct-memory"]; + + local bookieMemLimit = self["bookie-memory-limit"]; + local bookieMemReserv = self["bookie-memory-reservation"]; + local bookieHeap = self["bookie-heap"]; + local bookieDirect = self["bookie-direct-memory"]; + + local brokerMemLimit = self["broker-memory-limit"]; + local brokerMemReserv = self["broker-memory-reservation"]; + local brokerHeap = self["broker-heap"]; + local brokerDirect = self["broker-direct-memory"]; + + local initMemLimit = self["init-memory-limit"]; + local initMemReserv = self["init-memory-reservation"]; + local initHeap = self["init-heap"]; + local initDirect = self["init-direct-memory"]; + + // Zookeeper volume + local zkVolume = engine.volume("zookeeper").with_size("1G"); + + // Zookeeper container + local zkContainer = + engine.container("zookeeper") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/zookeeper.conf && bin/generate-zookeeper-config.sh conf/zookeeper.conf && exec bin/pulsar zookeeper" + ]) + .with_limits("1", zkMemLimit) + .with_reservations("0.05", zkMemReserv) + .with_user("0:1000") + .with_volume_mount(zkVolume, "/pulsar/data/zookeeper") + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + zkHeap, zkHeap, zkDirect, + ], + }) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Pulsar cluster init container + local initContainer = + engine.container("pulsar-init") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "sleep 10 && bin/pulsar initialize-cluster-metadata --cluster cluster-a --zookeeper zookeeper:2181 --configuration-store zookeeper:2181 --web-service-url http://pulsar:8080 --broker-service-url pulsar://pulsar:6650", + ]) + .with_limits("1", initMemLimit) + .with_reservations("0.05", initMemReserv) + .with_environment({ + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + initHeap, initHeap, initDirect, + ], + }); + + + // Bookkeeper volume + local bookieVolume = engine.volume("bookie").with_size("20G"); + + // Bookkeeper container + local bookieContainer = + engine.container("bookie") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/bookkeeper.conf && exec bin/pulsar bookie" + // false ^ causes this to be a 'failure' exit. + ]) + .with_limits("1", bookieMemLimit) + .with_reservations("0.1", bookieMemReserv) + .with_user("0:1000") + .with_volume_mount(bookieVolume, "/pulsar/data/bookkeeper") + .with_environment({ + "clusterName": "cluster-a", + "zkServers": "zookeeper:2181", + "bookieId": "bookie", + "metadataStoreUri": "metadata-store:zk:zookeeper:2181", + "advertisedAddress": "bookie", + "BOOKIE_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + bookieHeap, bookieHeap, bookieDirect, + ], + }) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker, stateless (uses ZK and Bookkeeper for state) + local brokerContainer = + engine.container("pulsar") + .with_image(images.pulsar) + .with_command([ + "bash", + "-c", + "bin/apply-config-from-env.py conf/broker.conf && exec bin/pulsar broker" + ]) + .with_limits("1", brokerMemLimit) + .with_reservations("0.1", brokerMemReserv) + .with_environment({ + "metadataStoreUrl": "zk:zookeeper:2181", + "zookeeperServers": "zookeeper:2181", + "clusterName": "cluster-a", + "managedLedgerDefaultEnsembleSize": "1", + "managedLedgerDefaultWriteQuorum": "1", + "managedLedgerDefaultAckQuorum": "1", + "advertisedAddress": "pulsar", + "advertisedListeners": "external:pulsar://pulsar:6650,localhost:pulsar://localhost:6650", + "PULSAR_MEM": "-Xms%s -Xmx%s -XX:MaxDirectMemorySize=%s" % [ + brokerHeap, brokerHeap, brokerDirect, + ], + }) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + // Container sets + local zkContainerSet = engine.containers( + "zookeeper", + [ + zkContainer, + ] + ); + + local initContainerSet = engine.containers( + "init-pulsar", + [ + initContainer, + ] + ); + + local bookieContainerSet = engine.containers( + "bookie", + [ + bookieContainer, + ] + ); + + local brokerContainerSet = engine.containers( + "pulsar", + [ + brokerContainer, + ] + ); + + // Zookeeper service + local zkService = + engine.service(zkContainerSet) + .with_port(2181, 2181, "zookeeper") + .with_port(2888, 2888, "zookeeper2") + .with_port(3888, 3888, "zookeeper3"); + + // Bookkeeper service + local bookieService = + engine.service(bookieContainerSet) + .with_port(3181, 3181, "bookie"); + + // Pulsar broker service + local brokerService = + engine.service(brokerContainerSet) + .with_port(6650, 6650, "pulsar") + .with_port(8080, 8080, "admin"); + + engine.resources([ + zkVolume, + bookieVolume, + zkContainerSet, + initContainerSet, + bookieContainerSet, + brokerContainerSet, + zkService, + bookieService, + brokerService, + ]) + + } + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/pubsub/rabbitmq.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/pubsub/rabbitmq.jsonnet new file mode 100644 index 00000000..600b14fc --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/pubsub/rabbitmq.jsonnet @@ -0,0 +1,78 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; + +// RabbitMQ messaging fabric configuration. +// Replaces Pulsar as the message broker for TrustGraph. + +{ + + "pub-sub-args":: [ + "--pubsub-backend", + "rabbitmq", + "--rabbitmq-host", + "rabbitmq", + ], + + "pub-sub-init-args":: [ + "--pulsar-admin-url", + url.pulsar_admin, + ], + + "overview-dashboard":: + importstr "grafana/dashboards/overview-dashboard-rabbitmq.json", + + "prometheus-config":: + importstr "prometheus/prometheus-rabbitmq.yml", + + "rabbitmq" +: { + + // Memory settings (can be overridden by memory-profile) + "memory-limit":: "1024M", + "memory-reservation":: "1024M", + + // CPU settings + "cpu-limit":: "1", + "cpu-reservation":: "0.1", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + // RabbitMQ volume for persistent data + local volume = engine.volume("rabbitmq").with_size("10G"); + + // RabbitMQ container + local container = + engine.container("rabbitmq") + .with_image(images.rabbitmq) + .with_limits(self["cpu-limit"], memoryLimit) + .with_reservations(self["cpu-reservation"], memoryReservation) + .with_volume_mount(volume, "/var/lib/rabbitmq") + .with_environment({ + "RABBITMQ_DEFAULT_USER": "guest", + "RABBITMQ_DEFAULT_PASS": "guest", + }) + .with_port(5672, 5672, "amqp") + .with_port(15672, 15672, "management") + .with_port(15692, 15692, "metrics"); + + local containerSet = engine.containers( + "rabbitmq", [ container ] + ); + + // RabbitMQ service + local service = + engine.service(containerSet) + .with_port(5672, 5672, "amqp") + .with_port(15672, 15672, "management"); + + engine.resources([ + volume, + containerSet, + service, + ]) + + } + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-additionals.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-additionals.jsonnet new file mode 100644 index 00000000..3e9f3481 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-additionals.jsonnet @@ -0,0 +1,134 @@ +local decode = import "decode-config.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Custom engine that collects configVolume parts +local engine = { + + // Collection of all configVolume parts + configVolumes:: [], + + // Implement all required engine methods as no-ops + container:: function(name) { + with_image:: function(x) self, + with_command:: function(x) self, + with_environment:: function(x) self, + with_limits:: function(c, m) self, + with_reservations:: function(c, m) self, + with_port:: function(src, dest, name) self, + with_volume_mount:: function(vol, mnt) self, + with_user:: function(x) self, + with_runtime:: function(x) self, + with_privileged:: function(x) self, + with_ipc:: function(x) self, + with_capability:: function(x) self, + with_device:: function(hdev, cdev) self, + with_env_var_secrets:: function(vars) self, + }, + + volume:: function(name) { + with_size:: function(size) self, + }, + + // The key method - collects configVolume parts + configVolume:: function(name, dir, parts) + local collector = self + { + configVolumes: super.configVolumes + [ + { + dir: dir, + parts: parts, + } + ] + }; + { + // Return a dummy volume that has the collector in it + name: name, + with_size:: function(size) collector, + // Provide a way to get back to the collector + getCollector:: function() collector, + }, + + secretVolume:: function(name, dir, parts) { + with_size:: function(size) self, + }, + + envSecrets:: function(name) { + with_env_var:: function(name, key) self, + }, + + containers:: function(name, containers) self, + + service:: function(containers) { + with_port:: function(src, dest, name) self, + }, + + internalService:: function(containers) { + with_port:: function(src, dest, name) self, + }, + + resources:: function(res) + // Fold over resources and collect any configVolume state + local collected = std.foldl( + function(state, r) + if std.objectHasAll(r, 'getCollector') then + // Merge the configVolumes from the volume's collector into our state + local volumeCollector = r.getCollector(); + state + { + configVolumes: state.configVolumes + volumeCollector.configVolumes + } + else + state, + res, + self + ); + collected, +}; + +// Execute all component create() functions with our collecting engine +// Note: create:: is a hidden field, so we must use objectHasAll not objectHas +local result = std.foldl( + function(state, p) + if std.objectHasAll(p, 'create') then + // Pattern has create directly - call it + p.create(state) + else + state, + std.objectValues(patterns), + engine +); + +// Debug: show what we collected +local debug = { + numPatterns: std.length(std.objectValues(patterns)), + numConfigVolumes: std.length(result.configVolumes), +}; + +// Transform collected data into output format +local allFiles = std.flattenArrays([ + [ + { + // Remove trailing slash from dir to avoid double slashes + path: std.join("/", [std.rstripChars(cv.dir, "/"), filename]), + content: cv.parts[filename] + } + for filename in std.objectFields(cv.parts) + ] + for cv in result.configVolumes +]); + +// Deduplicate by path - use a map to keep only unique paths +local uniqueMap = std.foldl( + function(acc, item) acc + { [item.path]: item }, + allFiles, + {} +); + +// Convert back to array +local additionals = std.objectValues(uniqueMap); + +// Output the array +additionals diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-aks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-aks-k8s.jsonnet new file mode 100644 index 00000000..bcec1cbb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-aks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/aks-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-docker-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-docker-compose.jsonnet new file mode 100644 index 00000000..7f0dbabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-docker-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/docker-compose.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-eks-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-eks-k8s.jsonnet new file mode 100644 index 00000000..f9de59cb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-eks-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/eks-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-gcp-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-gcp-k8s.jsonnet new file mode 100644 index 00000000..50037a5c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-gcp-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/gcp-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-minikube-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-minikube-k8s.jsonnet new file mode 100644 index 00000000..6fa64706 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-minikube-k8s.jsonnet @@ -0,0 +1,26 @@ + +local engine = import "../engine/minikube-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +local ns = { + apiVersion: "v1", + kind: "Namespace", + metadata: { + name: "trustgraph", + }, + "spec": { + }, +}; + +// Extract resources using the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-noop.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-noop.jsonnet new file mode 100644 index 00000000..3ad735d9 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-noop.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/noop.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-ovh-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-ovh-k8s.jsonnet new file mode 100644 index 00000000..92f61041 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-ovh-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/ovh-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-podman-compose.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-podman-compose.jsonnet new file mode 100644 index 00000000..7f0dbabe --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-podman-compose.jsonnet @@ -0,0 +1,20 @@ + +local engine = import "../engine/docker-compose.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resources = std.foldl( + function(state, p) state + p.create(engine), + std.objectValues(patterns), + {} +); + +resources + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-scw-k8s.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-scw-k8s.jsonnet new file mode 100644 index 00000000..4f6c1d7a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-scw-k8s.jsonnet @@ -0,0 +1,16 @@ + +local engine = import "../engine/scw-k8s.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract resources usnig the engine +local resourceList = engine.package(patterns); + +resourceList + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-tg-configuration.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-tg-configuration.jsonnet new file mode 100644 index 00000000..59b4732a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/config-to-tg-configuration.jsonnet @@ -0,0 +1,15 @@ + +local engine = import "../engine/noop.jsonnet"; +local decode = import "decode-config.jsonnet"; +local components = import "../components.jsonnet"; + +// Import config +local config = import "config.json"; + +// Produce patterns from config +local patterns = decode(config); + +// Extract configuration directly from patterns +patterns.configuration.configuration + + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/decode-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/decode-config.jsonnet new file mode 100644 index 00000000..759513c4 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/renderers/decode-config.jsonnet @@ -0,0 +1,31 @@ + +local components = import "../components.jsonnet"; + +local apply = function(p, components) + + local base = { + + with:: function(k, v) self + { + [k] +:: v + }, + + with_params:: function(pars) + self + std.foldl( + function(obj, par) obj.with(par.key, par.value), + std.objectKeysValues(pars), + self + ), + + }; + + local component = base + components[p.name]; + + component.with_params(p.parameters); + +local decode = function(config) + local add = function(state, c) state + apply(c, components); + local patterns = std.foldl(add, config, {}); + patterns; + +decode + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/row-store/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/row-store/cassandra.jsonnet new file mode 100644 index 00000000..f1b7adb8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/row-store/cassandra.jsonnet @@ -0,0 +1,75 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "backends/cassandra.jsonnet"; + +cassandra + { + + "store-rows" +: { + + create:: function(engine) + + local container = + engine.container("store-rows") + .with_image(images.trustgraph_flow) + .with_command([ + "rows-write-cassandra", +] + $["pub-sub-args"] + [ + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-rows", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-rows" +: { + + create:: function(engine) + + local container = + engine.container("query-rows") + .with_image(images.trustgraph_flow) + .with_command([ + "rows-query-cassandra", +] + $["pub-sub-args"] + [ + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-rows", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/agent-patterns.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/agent-patterns.jsonnet new file mode 100644 index 00000000..205e0cd8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/agent-patterns.jsonnet @@ -0,0 +1,20 @@ + +{ + "react": { + "name": "react", + "description": "Interleaved reasoning and action β€” the agent thinks, selects a tool, observes the result, and repeats until it has enough information to answer.", + "max_iterations": 10, + }, + "plan-then-execute": { + "name": "plan-then-execute", + "description": "The agent first produces a structured plan of steps, then executes each step in order. Supports re-planning on failure.", + "max_iterations": 15, + "max_replan_depth": 2, + }, + "supervisor": { + "name": "supervisor", + "description": "The agent decomposes the question into independent sub-goals, fans out subagent requests, waits for all to complete, then synthesises a final answer.", + "max_subagents": 5, + }, +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/agent-task-types.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/agent-task-types.jsonnet new file mode 100644 index 00000000..24fc3463 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/agent-task-types.jsonnet @@ -0,0 +1,27 @@ + +{ + "general": { + "name": "general", + "description": "No domain framing. All patterns are valid.", + "valid_patterns": ["react", "plan-then-execute", "supervisor"], + "framing": "", + }, + "research": { + "name": "research", + "description": "Research-oriented queries requiring information gathering.", + "valid_patterns": ["react", "plan-then-execute"], + "framing": "This is a research query. Focus on gathering accurate, well-sourced information.", + }, + "risk-assessment": { + "name": "risk-assessment", + "description": "Risk assessment queries requiring multi-dimensional analysis.", + "valid_patterns": ["supervisor", "plan-then-execute", "react"], + "framing": "This is a risk assessment query. Consider multiple risk dimensions and provide structured analysis.", + }, + "summarisation": { + "name": "summarisation", + "description": "Summarisation queries β€” straightforward information condensation.", + "valid_patterns": ["react"], + "framing": "This is a summarisation query. Focus on concise, accurate synthesis of the available information.", + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/config-composer.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/config-composer.jsonnet new file mode 100644 index 00000000..c1897e5b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/config-composer.jsonnet @@ -0,0 +1,101 @@ +// Configuration Composer Module +// Orchestrates the complete configuration building process +// Combines all components into the final TrustGraph configuration + +local flow_builder = import "flow-builder.jsonnet"; +local interface_builder = import "interface-builder.jsonnet"; + +{ + // Main function to build the complete configuration + build: function(config_spec) + // Extract configuration parameters + local flow_blueprints = config_spec.flow_blueprints; + local default_flow_blueprint = config_spec.default_flow_blueprint; + local default_flow_id = config_spec.default_flow_id; + local flow_init_parameters = config_spec.flow_init_parameters; + + // Build all processors for the default flow + local blueprint_processors = flow_builder.build_blueprint_processors( + flow_blueprints, + default_flow_blueprint, + flow_init_parameters + ); + + local flow_processors = flow_builder.build_flow_processors( + flow_blueprints, + default_flow_blueprint, + default_flow_id, + flow_init_parameters + ); + + // Combine processors into flow objects + local processor_array = blueprint_processors + flow_processors; + local flow_objects = flow_builder.build_flow_objects(processor_array); + local active_flows = flow_builder.merge_flow_objects(flow_objects); + + // Build interfaces for the default flow + local default_flow_interfaces = interface_builder.build_interfaces( + flow_blueprints, + default_flow_blueprint, + default_flow_id, + flow_init_parameters + ); + + // Return object with nested configuration (for backwards compatibility) + { + // Create function (for backwards compatibility) + create: function(engine) {}, + + // The actual configuration object + configuration: { + // Prompts configuration + prompt: { + "system": config_spec.prompts["system-template"], + "template-index": std.objectFieldsAll(config_spec.prompts.templates), + } + { + ["template." + template.key]: template.value + for template in std.objectKeysValuesAll(config_spec.prompts.templates) + }, + + // Tools configuration + tool: { + [tool.id]: tool + for tool in config_spec.tools + }, + + // MCP configuration + mcp: config_spec.mcp, + + // Agent orchestrator + "agent-pattern": config_spec.agent_patterns, + "agent-task-type": config_spec.agent_task_types, + + // Flow blueprints reference + "flow-blueprint": flow_blueprints, + + // Interface descriptions + "interface-description": config_spec.interface_descriptions, + + // Flow instances + "flow": { + [default_flow_id]: { + "description": "Default processing flow", + "blueprint-name": default_flow_blueprint, + "interfaces": default_flow_interfaces, + "parameters": flow_init_parameters, + }, + }, + + // Active flow processors + "active-flow": active_flows, + + // Token costs and parameter types + "token-cost": config_spec.token_costs, + "parameter-type": config_spec.parameter_types, + + // Collections configuration + "collection": config_spec.collection, + + }, + }, +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/flow-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/flow-builder.jsonnet new file mode 100644 index 00000000..0b142750 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/flow-builder.jsonnet @@ -0,0 +1,72 @@ +// Flow Builder Module +// Processes flow blueprints and builds complete flow configurations +// Handles {blueprint}, {id}, and parameter substitutions + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds blueprint-level processors with parameter substitution + // Processes the 'blueprint' section of flow blueprints + build_blueprint_processors: function(flow_blueprints, blueprint_name, parameters) + [ + [ + // Replace {blueprint} in the processor key + local key = std.strReplace(processor.key, "{blueprint}", blueprint_name); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // First replace {blueprint}, then substitute parameters + local blueprint_replaced = std.strReplace(field.value, "{blueprint}", blueprint_name); + param_processor.substitute_parameters(blueprint_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_blueprints[blueprint_name].blueprint) + ], + + // Builds flow-level processors with parameter substitution + // Processes the 'flow' section of flow blueprints + build_flow_processors: function(flow_blueprints, blueprint_name, flow_id, parameters) + [ + [ + // Replace both {blueprint} and {id} in the processor key + local key = std.strReplace( + std.strReplace(processor.key, "{blueprint}", blueprint_name), + "{id}", flow_id + ); + local parts = std.splitLimit(key, ":", 2); + parts, + { + // Process each field in the processor configuration + [field.key]: + // Replace {blueprint} and {id}, then substitute parameters + local blueprint_replaced = std.strReplace(field.value, "{blueprint}", blueprint_name); + local id_replaced = std.strReplace(blueprint_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(processor.value) + } + ] + for processor in std.objectKeysValuesAll(flow_blueprints[blueprint_name].flow) + ], + + // Combines blueprint and flow processors into flow objects + build_flow_objects: function(processor_array) + std.map( + function(item) { + [item[0][0]] +: { + [item[0][1]]: item[1] + } + }, + processor_array + ), + + // Merges all flow objects into a single flows_active configuration + merge_flow_objects: function(flow_objects) + std.foldr( + function(a, b) a + b, + flow_objects, + {} + ), +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/interface-builder.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/interface-builder.jsonnet new file mode 100644 index 00000000..2143e600 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/interface-builder.jsonnet @@ -0,0 +1,30 @@ +// Interface Builder Module +// Processes flow class interfaces with parameter substitution +// Handles both string interfaces and nested object interfaces + +local param_processor = import "parameter-processor.jsonnet"; + +{ + // Builds interfaces for a specific flow class and instance + // Processes the 'interfaces' section of flow classes + build_interfaces: function(flow_classes, class_name, flow_id, parameters) + local interface_spec = flow_classes[class_name].interfaces; + { + [interface.key]: + if std.isString(interface.value) then + // Simple string interface - apply all substitutions + local class_replaced = std.strReplace(interface.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + else + // Complex object interface - process nested fields + { + [field.key]: + local class_replaced = std.strReplace(field.value, "{class}", class_name); + local id_replaced = std.strReplace(class_replaced, "{id}", flow_id); + param_processor.substitute_parameters(id_replaced, parameters) + for field in std.objectKeysValuesAll(interface.value) + } + for interface in std.objectKeysValuesAll(interface_spec) + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/interface-descriptions.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/interface-descriptions.jsonnet new file mode 100644 index 00000000..f9dda8b0 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/interface-descriptions.jsonnet @@ -0,0 +1,89 @@ +// Interface Descriptions Module +// Defines all external interfaces available in TrustGraph flows +// These are the 'endpoints' that external systems can interact with + +{ + // Document loading interfaces - for data ingestion + "document-load": { + "description": "Document loader", + "kind": "send", + "visible": true, + }, + "text-load": { + "description": "Text document loader", + "kind": "send", + "visible": true, + }, + + // Data storage interfaces - for processed data streams + "entity-contexts-load": { + "description": "Entity contexts loader", + "kind": "send", + }, + "triples-store": { + "description": "Triples loader", + "kind": "send", + }, + "graph-embeddings-store": { + "description": "Graph embeddings loader", + "kind": "send", + }, + "document-embeddings-store": { + "description": "Document embeddings loader", + "kind": "send", + }, + "objects-store": { + "description": "Object store", + "kind": "request-response", + }, + + // Query interfaces - for retrieving information + "graph-rag": { + "description": "GraphRAG service", + "kind": "request-response", + }, + "document-rag": { + "description": "ChunkRAG service", + "kind": "request-response", + }, + "triples": { + "description": "Triples query service", + "kind": "request-response", + }, + "graph-embeddings": { + "description": "Graph embeddings service", + "kind": "request-response", + }, + "document-embeddings": { + "description": "Document embeddings service", + "kind": "request-response", + }, + "objects": { + "description": "Object query service", + "kind": "request-response", + }, + + // Processing services - for text and data processing + "prompt": { + "description": "Prompt service", + "kind": "request-response", + }, + "agent": { + "description": "Agent service", + "kind": "request-response", + }, + "text-completion": { + "description": "Text completion service", + "kind": "request-response", + }, + + // Query translation services - for natural language queries + "nlp-query": { + "description": "NLP question to GraphQL service", + "kind": "request-response", + }, + "structured-query": { + "description": "Structured query service", + "kind": "request-response", + }, +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/parameter-processor.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/parameter-processor.jsonnet new file mode 100644 index 00000000..f317ae9f --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/parameter-processor.jsonnet @@ -0,0 +1,38 @@ +// Parameter Processing Module +// Handles dynamic parameter replacement in configuration values +// Replaces {parameter_name} placeholders with actual parameter values + +{ + // Applies parameter substitutions to string values + // Only processes strings - leaves other types unchanged + substitute_parameters: function(value, parameters) + if std.isString(value) then + std.foldl( + function(acc, param) + // Only do string replacement if param.value is a string + if std.isString(param.value) then + std.strReplace(acc, "{" + param.key + "}", param.value) + else + acc, // Skip replacement for non-string parameter values + std.objectKeysValuesAll(parameters), + value + ) + else + value, + + // Applies parameter substitutions to all values in an object + // Recursively processes nested objects and arrays + substitute_parameters_in_object: function(obj, parameters) + if std.isObject(obj) then + { + [key]: $.substitute_parameters_in_object(obj[key], parameters) + for key in std.objectFields(obj) + } + else if std.isArray(obj) then + [ + $.substitute_parameters_in_object(item, parameters) + for item in obj + ] + else + $.substitute_parameters(obj, parameters), +} \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/tools.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/tools.jsonnet new file mode 100644 index 00000000..4947279e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/tools.jsonnet @@ -0,0 +1,36 @@ +// Tools Configuration Module +// Defines all available tools that can be used by agents and flows +// Each tool specifies its interface, arguments, and behavior + +[ + // Knowledge query tool - queries the knowledge base + { + id: "knowledge-query", + name: "Knowledge query", + description: "This tool queries a knowledge base that holds information about domain-specific information. The question should be a natural language question.", + type: "knowledge-query", + collection: "default", + arguments: [ + { + name: "question", + type: "string", + description: "A simple natural language question.", + } + ] + }, + + // LLM completion tool - general purpose text completion + { + id: "llm-completion", + name: "LLM text completion", + type: "text-completion", + description: "This tool queries an LLM for non-domain-specific information. The question should be a natural language question.", + arguments: [ + { + name: "question", + type: "string", + description: "The question which should be asked of the LLM.", + } + ] + } +] \ No newline at end of file diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/trustgraph-config.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/trustgraph-config.jsonnet new file mode 100644 index 00000000..9d899ceb --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/runtime-config/trustgraph-config.jsonnet @@ -0,0 +1,96 @@ +// TrustGraph Main Configuration +// Clean, modular composition of TrustGraph configuration +// Uses specialized modules for different aspects of config building + +// Import dependencies +local images = import "../values/images.jsonnet"; +local url = import "../values/url.jsonnet"; +local prompts = import "../prompts/mixtral.jsonnet"; +local default_prompts = import "../prompts/default-prompts.jsonnet"; +local token_costs = import "../values/token-costs.jsonnet"; +local flow_blueprints = import "../flows/flow-blueprints.jsonnet"; +local config_composer = import "config-composer.jsonnet"; +local interface_descriptions = import "interface-descriptions.jsonnet"; +local tools = import "tools.jsonnet"; +local temperature_params = import "../parameters/temperature-param-types.jsonnet"; +local chunking_params = import "../parameters/chunking-param-types.jsonnet"; + +// Main configuration object +local configuration = { + + // Prompt templates + prompts:: default_prompts, + + // Tool definitions + tools:: tools, + + // MCP configuration + mcp:: {}, + + // Flow classes reference + "flow-blueprints":: flow_blueprints, + + // LLM model parameters + "llm-models" +:: {}, + + // Embeddings model parameters + "embeddings-models" +:: {}, + + collections +:: { + "trustgraph:default": { + "user": "default-user", + "collection": "default", + "name": "Default Collection", + "description": "Default collection", + "tags": ["default"], + }, + }, + + // Default model and flow parameters + flow_init_parameters:: { + "llm-model": $["llm-models"].default, + "llm-rag-model": $["llm-models"].default, + "llm-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "llm-rag-temperature": "%0.3f" % $["parameter-types"]["llm-temperature"].default, + "chunk-size": std.toString($["parameter-types"]["chunk-size"].default), + "chunk-overlap": std.toString($["parameter-types"]["chunk-overlap"].default), + "embeddings-model": $["embeddings-models"].default, + }, + + // Interface descriptions for external endpoints + "interface-descriptions":: interface_descriptions, + + // Parameter type definitions + "parameter-types":: { + "llm-model": $["llm-models"], + "embeddings-model": $["embeddings-models"], + } + chunking_params + temperature_params, + + // Token costs + "token-costs":: token_costs, + + // Agent orchestator configuration + "agent-patterns":: import "agent-patterns.jsonnet", + "agent-task-types":: import "agent-task-types.jsonnet", + + // Build the complete configuration using the composer + configuration:: config_composer.build({ + flow_blueprints: $["flow-blueprints"], + default_flow_blueprint: "everything", + default_flow_id: "default", + flow_init_parameters: $["flow_init_parameters"], + prompts: $["prompts"], + tools: $["tools"], + mcp: $["mcp"], + interface_descriptions: $["interface-descriptions"], + parameter_types: $["parameter-types"], + token_costs: $["token-costs"], + collection: $["collections"], + agent_patterns: $["agent-patterns"], + agent_task_types: $["agent-task-types"], + }), + +} + default_prompts; + +// Export the final configuration +configuration diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/cassandra.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/cassandra.jsonnet new file mode 100644 index 00000000..d7a8460c --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/cassandra.jsonnet @@ -0,0 +1,75 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local cassandra = import "backends/cassandra.jsonnet"; + +cassandra + { + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-cassandra", +] + $["pub-sub-args"] + [ + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "256M") + .with_reservations("0.1", "256M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-cassandra", +] + $["pub-sub-args"] + [ + "--cassandra-host", + cassandra_hosts, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "512M") + .with_reservations("0.1", "512M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/falkordb.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/falkordb.jsonnet new file mode 100644 index 00000000..7fac275e --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/falkordb.jsonnet @@ -0,0 +1,77 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local falkordb = import "backends/falkordb.jsonnet"; + +falkordb + { + + "falkordb-url":: "falkor://falkordb:6379", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-falkordb", +] + $["pub-sub-args"] + [ + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-falkordb", +] + $["pub-sub-args"] + [ + "-g", + $["falkordb-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/memgraph.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/memgraph.jsonnet new file mode 100644 index 00000000..06003fac --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/memgraph.jsonnet @@ -0,0 +1,82 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local memgraph = import "backends/memgraph.jsonnet"; + +memgraph + { + + "memgraph-url":: "bolt://memgraph:7687", + "memgraph-database":: "memgraph", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-memgraph", +] + $["pub-sub-args"] + [ + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-memgraph", +] + $["pub-sub-args"] + [ + "-g", + $["memgraph-url"], + "--database", + $["memgraph-database"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/neo4j.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/neo4j.jsonnet new file mode 100644 index 00000000..cca8500a --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/triple-store/neo4j.jsonnet @@ -0,0 +1,77 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local neo4j = import "backends/neo4j.jsonnet"; + +neo4j + { + + "neo4j-url":: "bolt://neo4j:7687", + + "store-triples" +: { + + create:: function(engine) + + local container = + engine.container("store-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-write-neo4j", +] + $["pub-sub-args"] + [ + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-triples" +: { + + create:: function(engine) + + local container = + engine.container("query-triples") + .with_image(images.trustgraph_flow) + .with_command([ + "triples-query-neo4j", +] + $["pub-sub-args"] + [ + "-g", + $["neo4j-url"], + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-triples", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/ui/workbench-ui.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/ui/workbench-ui.jsonnet new file mode 100644 index 00000000..f2048e47 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/ui/workbench-ui.jsonnet @@ -0,0 +1,32 @@ +local images = import "values/images.jsonnet"; + +{ + + "workbench-ui" +: { + + create:: function(engine) + + local container = + engine.container("workbench-ui") + .with_image(images["workbench-ui"]) + .with_limits("0.1", "256M") + .with_reservations("0.1", "256M") + .with_port(8888, 8888, "ui"); + + local containerSet = engine.containers( + "workbench-ui", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8888, 8888, "ui"); + + engine.resources([ + containerSet, + service, + ]) + + }, + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/values/images.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/values/images.jsonnet new file mode 100644 index 00000000..3b29db46 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/values/images.jsonnet @@ -0,0 +1,38 @@ +local version = import "version.jsonnet"; +{ + cassandra: "docker.io/cassandra:5.0.6", +// Not working +// ceph: "quay.io/ceph/daemon:latest-reef", + neo4j: "docker.io/neo4j:2025.08.0-community-bullseye", + pulsar: "docker.io/apachepulsar/pulsar:4.1.0", + rabbitmq: "docker.io/rabbitmq:4.1-management", + pulsar_manager: "docker.io/apachepulsar/pulsar-manager:v0.4.0", + etcd: "quay.io/coreos/etcd:v3.6.4", + minio: "docker.io/minio/minio:RELEASE.2025-09-07T16-13-09Z", + garage: "docker.io/dxflrs/garage:v2.1.0", + milvus: "docker.io/milvusdb/milvus:v2.5.17", + prometheus: "docker.io/prom/prometheus:v3.8.0", + grafana: "docker.io/grafana/grafana:12.3.0", + loki: "docker.io/grafana/loki:3.6.2", + trustgraph_base: "docker.io/trustgraph/trustgraph-base:" + version, + trustgraph_flow: "docker.io/trustgraph/trustgraph-flow:" + version, + trustgraph_ocr: "docker.io/trustgraph/trustgraph-ocr:" + version, + trustgraph_bedrock: "docker.io/trustgraph/trustgraph-bedrock:" + version, + trustgraph_vertexai: "docker.io/trustgraph/trustgraph-vertexai:" + version, + trustgraph_hf: "docker.io/trustgraph/trustgraph-hf:" + version, + trustgraph_mcp: "docker.io/trustgraph/trustgraph-mcp:" + version, + trustgraph_unstructured: "docker.io/trustgraph/trustgraph-unstructured:" + version, + qdrant: "docker.io/qdrant/qdrant:v1.15.4", + memgraph_mage: "docker.io/memgraph/memgraph-mage:3.5", + memgraph_lab: "docker.io/memgraph/lab:3.5.0", + falkordb: "docker.io/falkordb/falkordb:v4.12.5", + "workbench-ui": "docker.io/trustgraph/workbench-ui:1.7.2", + "ddg-mcp-server": "docker.io/trustgraph/ddg-mcp-server:0.1.0", + "tgi-service-intel-xpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-xpu", + "tgi-service-cpu": "ghcr.io/huggingface/text-generation-inference:3.3.1-intel-cpu", + "tgi-service-gaudi": "ghcr.io/huggingface/text-generation-inference:sha-f140440-gaudi", + "vllm-service-intel-xpu": "docker.io/intel/vllm:0.8.0-xpu", + "vllm-service-gaudi": "docker.io/trustgraph/vllm-hpu:027f5645", + "vllm-service-nvidia": "docker.io/vllm/vllm-openai:latest", + "vllm-service-intel-battlemage": "docker.io/intelanalytics/ipex-llm-serving-xpu:0.2.0-b6", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/values/token-costs.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/values/token-costs.jsonnet new file mode 100644 index 00000000..84c70373 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/values/token-costs.jsonnet @@ -0,0 +1,102 @@ +{ + "mistral.mistral-large-2407-v1:0": { + "model_name": "mistral.mistral-large-2407-v1:0", + "input_price": 0.000004, + "output_price": 0.000012 + }, + "meta.llama3-1-405b-instruct-v1:0": { + "model_name": "meta.llama3-1-405b-instruct-v1:0", + "input_price": 0.00000532, + "output_price": 0.000016 + }, + "mistral.mixtral-8x7b-instruct-v0:1": { + "model_name": "mistral.mixtral-8x7b-instruct-v0:1", + "input_price": 0.00000045, + "output_price": 0.0000007 + }, + "meta.llama3-1-70b-instruct-v1:0": { + "model_name": "meta.llama3-1-70b-instruct-v1:0", + "input_price": 0.00000099, + "output_price": 0.00000099 + }, + "meta.llama3-1-8b-instruct-v1:0": { + "model_name": "meta.llama3-1-8b-instruct-v1:0", + "input_price": 0.00000022, + "output_price": 0.00000022 + }, + "anthropic.claude-3-haiku-20240307-v1:0": { + "model_name": "anthropic.claude-3-haiku-20240307-v1:0", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "anthropic.claude-3-5-sonnet-20240620-v1:0": { + "model_name": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "cohere.command-r-plus-v1:0": { + "model_name": "cohere.command-r-plus-v1:0", + "input_price": 0.0000030, + "output_price": 0.0000150 + }, + "ollama": { + "model_name": "ollama", + "input_price": 0, + "output_price": 0 + }, + "claude-3-haiku-20240307": { + "model_name": "claude-3-haiku-20240307", + "input_price": 0.00000025, + "output_price": 0.00000125 + }, + "claude-3-5-sonnet-20240620": { + "model_name": "claude-3-5-sonnet-20240620", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "claude-3-opus-20240229": { + "model_name": "claude-3-opus-20240229", + "input_price": 0.000015, + "output_price": 0.000075 + }, + "claude-3-sonnet-20240229": { + "model_name": "claude-3-sonnet-20240229", + "input_price": 0.000003, + "output_price": 0.000015 + }, + "command-r-08-202": { + "model_name": "command-r-08-202", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "c4ai-aya-23-8b": { + "model_name": "c4ai-aya-23-8b", + "input_price": 0, + "output_price": 0 + }, + "llama.cpp": { + "model_name": "llama.cpp", + "input_price": 0, + "output_price": 0 + }, + "gpt-4o": { + "model_name": "gpt-4o", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-2024-08-06": { + "model_name": "gpt-4o-2024-08-06", + "input_price": 0.0000025, + "output_price": 0.000010 + }, + "gpt-4o-2024-05-13": { + "model_name": "gpt-4o-2024-05-13", + "input_price": 0.000005, + "output_price": 0.000015 + }, + "gpt-4o-mini": { + "model_name": "gpt-4o-mini", + "input_price": 0.00000015, + "output_price": 0.0000006 + } +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/values/url.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/values/url.jsonnet new file mode 100644 index 00000000..68f8f706 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/values/url.jsonnet @@ -0,0 +1,8 @@ +{ + pulsar: "pulsar://pulsar:6650", + pulsar_admin: "http://pulsar:8080", + amqp: "amqp://guest:guest@rabbitmq:5672", + milvus: "http://milvus:19530", + qdrant: "http://qdrant:6333", + object_store: "garage:3900", +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/vector-store/milvus.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/vector-store/milvus.jsonnet new file mode 100644 index 00000000..81b3380b --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/vector-store/milvus.jsonnet @@ -0,0 +1,142 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local milvus = import "backends/milvus.jsonnet"; + +milvus + { + + "store-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-milvus", +] + $["pub-sub-args"] + [ + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-milvus", +] + $["pub-sub-args"] + [ + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-milvus", +] + $["pub-sub-args"] + [ + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-milvus", +] + $["pub-sub-args"] + [ + "-t", + url.milvus, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/vector-store/pinecone.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/vector-store/pinecone.jsonnet new file mode 100644 index 00000000..ef26dfae --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/vector-store/pinecone.jsonnet @@ -0,0 +1,156 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; + +{ + + "pinecone-cloud":: "aws", + "pinecone-region":: "us-east-1", + + "store-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-write-pinecone", +] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "ge-query-pinecone", +] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-write-pinecone", +] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + create:: function(engine) + + local envSecrets = engine.envSecrets("pinecone-api-key") + .with_env_var("PINECONE_API_KEY", "pinecone-api-key"); + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "de-query-pinecone", +] + $["pub-sub-args"] + [ + "--log-level", + $["log-level"], + ]) + .with_env_var_secrets(envSecrets) + .with_limits("0.5", "128M") + .with_reservations("0.1", "128M"); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + envSecrets, + containerSet, + service, + ]) + + + } + +} + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/vector-store/qdrant.jsonnet b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/vector-store/qdrant.jsonnet new file mode 100644 index 00000000..4a2b84f8 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/vector-store/qdrant.jsonnet @@ -0,0 +1,244 @@ +local images = import "values/images.jsonnet"; +local url = import "values/url.jsonnet"; +local cassandra_hosts = "cassandra"; +local qdrant = import "backends/qdrant.jsonnet"; + +qdrant + { + + "store-graph-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings-write-qdrant", +] + $["pub-sub-args"] + [ + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-graph-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-graph-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "graph-embeddings-query-qdrant", +] + $["pub-sub-args"] + [ + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-graph-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "store-doc-embeddings" +: { + + "memory-limit":: "256M", + "memory-reservation":: "256M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "doc-embeddings-write-qdrant", +] + $["pub-sub-args"] + [ + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-doc-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-doc-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "doc-embeddings-query-qdrant", +] + $["pub-sub-args"] + [ + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-doc-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + }, + + "store-row-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("store-row-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "row-embeddings-write-qdrant", +] + $["pub-sub-args"] + [ + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "store-row-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + }, + + "query-row-embeddings" +: { + + "memory-limit":: "128M", + "memory-reservation":: "128M", + + create:: function(engine) + + local memoryLimit = self["memory-limit"]; + local memoryReservation = self["memory-reservation"]; + + local container = + engine.container("query-row-embeddings") + .with_image(images.trustgraph_flow) + .with_command([ + "row-embeddings-query-qdrant", +] + $["pub-sub-args"] + [ + "-t", + url.qdrant, + "--log-level", + $["log-level"], + ]) + .with_limits("0.5", memoryLimit) + .with_reservations("0.1", memoryReservation); + + local containerSet = engine.containers( + "query-row-embeddings", [ container ] + ); + + local service = + engine.internalService(containerSet) + .with_port(8000, 8000, "metrics"); + + engine.resources([ + containerSet, + service, + ]) + + + }, + +} diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/zip-readme.md b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/zip-readme.md new file mode 100644 index 00000000..0b117792 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/2.2/zip-readme.md @@ -0,0 +1,28 @@ + +Note! this is a subset of possible configurations, to generate your own +launch config use the config util... + +- Production: https://config-ui.demo.trustgraph.ai +- Early release: https://dev.config-ui.demo.trustgraph.ai + +The config util auto-generates deployment instructions for your +configuration, so that's the recommended way to deploy. + +---------------------------------------------------------------------------- + +These are launch configurations for TrustGraph. See https://trustgraph.ai for +the quickstart using docker compose. + +Hint for Linux: There are files here which get mounted as volumes inside +Docker Compose containers. This may trigger SELinux rules on your system, to +permit access insider the containers, use a command like this... + +chcon -Rt svirt_sandbox_file_t grafana/ prometheus/ + +The file vertexai/private.json is a placeholder for real GCP credentials if +you are using the VertexAI LLM. If you're using that in Docker Compose, +replace with your real credentials, and don't forget to permit access if you +are using Linux: + +chcon -Rt svirt_sandbox_file_t vertexai/ + diff --git a/ai-context/trustgraph-templates/trustgraph_configurator/templates/index.json b/ai-context/trustgraph-templates/trustgraph_configurator/templates/index.json new file mode 100644 index 00000000..dc94ed14 --- /dev/null +++ b/ai-context/trustgraph-templates/trustgraph_configurator/templates/index.json @@ -0,0 +1,80 @@ +{ + "templates": [ + { + "name": "1.8", + "description": "Pluggable messaging fabric, switched to Garage for object store", + "version": "1.8.21", + "status": "stable" + }, + { + "name": "1.9", + "description": "Ontology extraction update, JSONL output from prompts", + "version": "1.9.4", + "status": "beta" + }, + { + "name": "2.1", + "description": "End-to-end provenance tracking and explainability across the knowledge pipeline β€” from document ingestion through extraction to query-time retrieval - every answer traceable to its source evidence", + "version": "2.1.26", + "status": "stable" + }, + { + "name": "2.2", + "description": "New agent orchestration engine, extended document processing to handle more document types include MS Office types", + "version": "2.2.16", + "status": "pre-release" + } + ], + "platforms": [ + { + "name": "podman-compose", + "description": "Uses podman-compose to deploy a compose file to a local Podman environment" + }, + { + "name": "docker-compose", + "description": "Uses docker-compose to deploy a compose file to a local Docker installation" + }, + { + "name": "minikube-k8s", + "description": "Uses kubectl to deploy resources to a Minikube Kubernetes cluster" + }, + { + "name": "scw-k8s", + "description": "Uses kubectl to deploy resources to a Kubernetes cluster running in Scaleway cloud" + }, + { + "name": "ovh-k8s", + "description": "Uses kubectl to deploy resources to a Kubernetes cluster running in OVHCloud" + }, + { + "name": "gcp-k8s", + "description": "Uses kubectl to deploy resources to a Kubernetes cluster on Google Cloud (e.g. GKE)" + }, + { + "name": "aks-k8s", + "description": "Uses kubectl to deploy resources to a Kubernetes cluster in Azure (AKS)" + }, + { + "name": "eks-k8s", + "description": "Uses kubectl to deploy resources to an Kubernetes cluster running in AWS (e.g. EKS)" + } + ], + "statuses": [ + { + "name": "pre-release", + "description": "Pre-release" + }, + { + "name": "alpha", + "description": "Alpha" + }, + { + "name": "beta", + "description": "Beta" + }, + { + "name": "stable", + "description": "Stable for production use" + } + ] +} diff --git a/ai-context/trustgraph-templates/version.py b/ai-context/trustgraph-templates/version.py new file mode 100644 index 00000000..87fff4ab --- /dev/null +++ b/ai-context/trustgraph-templates/version.py @@ -0,0 +1,2 @@ +__version__ = "0.0.0" +