mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-01 09:29:38 +02:00
refactor(ts): make maintenance scripts effect native
This commit is contained in:
parent
a7bdbb9257
commit
cd6c9107d7
7 changed files with 845 additions and 475 deletions
|
|
@ -5,8 +5,11 @@
|
||||||
* extractor can identify. Writes to data/test.pdf.
|
* extractor can identify. Writes to data/test.pdf.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { BunRuntime } from "@effect/platform-bun";
|
||||||
|
import * as BunFileSystem from "@effect/platform-bun/BunFileSystem";
|
||||||
|
import { Effect } from "effect";
|
||||||
|
import * as FileSystem from "effect/FileSystem";
|
||||||
import { PDFDocument, StandardFonts } from "pdf-lib";
|
import { PDFDocument, StandardFonts } from "pdf-lib";
|
||||||
import { writeFileSync, mkdirSync } from "node:fs";
|
|
||||||
|
|
||||||
const PAGE_1 = `Acme Corporation: Company Overview
|
const PAGE_1 = `Acme Corporation: Company Overview
|
||||||
|
|
||||||
|
|
@ -24,10 +27,11 @@ CloudSync was officially launched in January 2024. The platform competes with es
|
||||||
|
|
||||||
Acme Corporation is headquartered in San Francisco, California. The company employs approximately 200 people across engineering, sales, and operations departments.`;
|
Acme Corporation is headquartered in San Francisco, California. The company employs approximately 200 people across engineering, sales, and operations departments.`;
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
const main = Effect.fn("createTestPdf.main")(function*() {
|
||||||
const pdf = await PDFDocument.create();
|
const fs = yield* FileSystem.FileSystem;
|
||||||
const font = await pdf.embedFont(StandardFonts.Helvetica);
|
const pdf = yield* Effect.promise(() => PDFDocument.create());
|
||||||
const boldFont = await pdf.embedFont(StandardFonts.HelveticaBold);
|
const font = yield* Effect.promise(() => pdf.embedFont(StandardFonts.Helvetica));
|
||||||
|
const boldFont = yield* Effect.promise(() => pdf.embedFont(StandardFonts.HelveticaBold));
|
||||||
|
|
||||||
for (const [i, text] of [PAGE_1, PAGE_2].entries()) {
|
for (const [i, text] of [PAGE_1, PAGE_2].entries()) {
|
||||||
const page = pdf.addPage([612, 792]); // US Letter
|
const page = pdf.addPage([612, 792]); // US Letter
|
||||||
|
|
@ -54,14 +58,11 @@ async function main(): Promise<void> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const pdfBytes = await pdf.save();
|
const pdfBytes = yield* Effect.promise(() => pdf.save());
|
||||||
|
|
||||||
mkdirSync("data", { recursive: true });
|
yield* fs.makeDirectory("data", { recursive: true });
|
||||||
writeFileSync("data/test.pdf", pdfBytes);
|
yield* fs.writeFile("data/test.pdf", pdfBytes);
|
||||||
console.log(`Created data/test.pdf (${pdfBytes.length} bytes, 2 pages)`);
|
console.log(`Created data/test.pdf (${pdfBytes.length} bytes, 2 pages)`);
|
||||||
}
|
|
||||||
|
|
||||||
main().catch((err) => {
|
|
||||||
console.error("Failed to create test PDF:", err);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
BunRuntime.runMain(main().pipe(Effect.provide(BunFileSystem.layer)));
|
||||||
|
|
|
||||||
|
|
@ -1 +1 @@
|
||||||
{"exemptions":[],"baseline":[{"rule":"no-error-throw","path":"packages/workbench/src/main.tsx","count":1},{"rule":"no-error-throw","path":"scripts/seed-config.ts","count":1},{"rule":"no-error-throw","path":"scripts/seed-demo.ts","count":4},{"rule":"no-error-throw","path":"scripts/seed-flows.ts","count":2},{"rule":"no-error-throw","path":"scripts/test-pipeline.ts","count":2},{"rule":"no-native-fetch","path":"scripts/seed-config.ts","count":1},{"rule":"no-native-fetch","path":"scripts/seed-demo.ts","count":11},{"rule":"no-native-fetch","path":"scripts/seed-flows.ts","count":1},{"rule":"no-native-fetch","path":"scripts/test-pipeline.ts","count":5},{"rule":"no-native-json","path":"scripts/seed-config.ts","count":6},{"rule":"no-native-json","path":"scripts/seed-demo.ts","count":6},{"rule":"no-native-json","path":"scripts/seed-flows.ts","count":3},{"rule":"no-native-json","path":"scripts/test-pipeline.ts","count":6},{"rule":"no-native-sort","path":"packages/client/src/socket/trustgraph-socket.ts","count":2},{"rule":"no-native-sort","path":"packages/flow/src/config/service.ts","count":3},{"rule":"no-native-sort","path":"packages/flow/src/cores/service.ts","count":1},{"rule":"no-native-sort","path":"packages/flow/src/flow-manager/service.ts","count":1},{"rule":"no-native-sort","path":"packages/flow/src/librarian/service.ts","count":1},{"rule":"no-native-sort","path":"packages/flow/src/retrieval/graph-rag.ts","count":1},{"rule":"no-native-sort","path":"packages/mcp/src/server-effect.ts","count":1},{"rule":"no-native-sort","path":"packages/workbench/src/atoms/workbench.ts","count":2},{"rule":"no-native-sort","path":"packages/workbench/src/components/chat/explain-graph.tsx","count":1},{"rule":"no-native-sort","path":"packages/workbench/src/pages/graph.tsx","count":1},{"rule":"no-native-sort","path":"packages/workbench/src/qa/mock-api.ts","count":1},{"rule":"no-native-sort","path":"scripts/inventory-native-classes.ts","count":1},{"rule":"no-native-sort","path":"scripts/seed-demo.ts","count":1},{"rule":"no-native-sort","path":"scripts/seed-flows.ts","count":1},{"rule":"no-native-timers","path":"scripts/test-pipeline.ts","count":2},{"rule":"no-node-fs-path","path":"scripts/create-test-pdf.ts","count":1},{"rule":"no-node-fs-path","path":"scripts/inventory-native-classes.ts","count":2},{"rule":"no-process-env","path":"scripts/seed-config.ts","count":2},{"rule":"no-process-env","path":"scripts/seed-demo.ts","count":5},{"rule":"no-process-env","path":"scripts/seed-flows.ts","count":1},{"rule":"no-process-env","path":"scripts/test-pipeline.ts","count":11},{"rule":"no-schema-suffix","path":"packages/base/src/schema/primitives.ts","count":1}]}
|
{"exemptions":[],"baseline":[{"rule":"no-error-throw","path":"packages/workbench/src/main.tsx","count":1},{"rule":"no-native-sort","path":"packages/client/src/socket/trustgraph-socket.ts","count":2},{"rule":"no-native-sort","path":"packages/flow/src/config/service.ts","count":3},{"rule":"no-native-sort","path":"packages/flow/src/cores/service.ts","count":1},{"rule":"no-native-sort","path":"packages/flow/src/flow-manager/service.ts","count":1},{"rule":"no-native-sort","path":"packages/flow/src/librarian/service.ts","count":1},{"rule":"no-native-sort","path":"packages/flow/src/retrieval/graph-rag.ts","count":1},{"rule":"no-native-sort","path":"packages/mcp/src/server-effect.ts","count":1},{"rule":"no-native-sort","path":"packages/workbench/src/atoms/workbench.ts","count":2},{"rule":"no-native-sort","path":"packages/workbench/src/components/chat/explain-graph.tsx","count":1},{"rule":"no-native-sort","path":"packages/workbench/src/pages/graph.tsx","count":1},{"rule":"no-native-sort","path":"packages/workbench/src/qa/mock-api.ts","count":1},{"rule":"no-schema-suffix","path":"packages/base/src/schema/primitives.ts","count":1}]}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,8 @@
|
||||||
import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
|
import { BunRuntime } from "@effect/platform-bun";
|
||||||
import { join, relative, sep } from "node:path";
|
import * as BunFileSystem from "@effect/platform-bun/BunFileSystem";
|
||||||
|
import { Array as A, Effect, Layer, Order, Path, Schema as S } from "effect";
|
||||||
|
import * as FileSystem from "effect/FileSystem";
|
||||||
|
import type { PlatformError } from "effect/PlatformError";
|
||||||
import ts from "typescript";
|
import ts from "typescript";
|
||||||
|
|
||||||
type Scope = "production" | "non-production";
|
type Scope = "production" | "non-production";
|
||||||
|
|
@ -17,10 +20,8 @@ interface ClassFinding {
|
||||||
}
|
}
|
||||||
|
|
||||||
const root = process.cwd();
|
const root = process.cwd();
|
||||||
const packagesSrc = join(root, "packages");
|
|
||||||
const scriptsDir = join(root, "scripts");
|
|
||||||
|
|
||||||
const sourceExtensions = new Set([".ts", ".tsx", ".mts", ".cts"]);
|
const sourceExtensions = new Set([".ts", ".tsx", ".mts", ".cts"]);
|
||||||
|
const ignoredDirectories = new Set(["dist", "node_modules", ".turbo"]);
|
||||||
const effectClassPatterns = [
|
const effectClassPatterns = [
|
||||||
/\bS\.(Class|TaggedClass|TaggedErrorClass|ErrorClass)\b/,
|
/\bS\.(Class|TaggedClass|TaggedErrorClass|ErrorClass)\b/,
|
||||||
/\bSchema\.(Class|TaggedClass|TaggedErrorClass|ErrorClass)\b/,
|
/\bSchema\.(Class|TaggedClass|TaggedErrorClass|ErrorClass)\b/,
|
||||||
|
|
@ -33,6 +34,11 @@ const effectClassPatterns = [
|
||||||
/\bEffect\.Service\b/,
|
/\bEffect\.Service\b/,
|
||||||
];
|
];
|
||||||
|
|
||||||
|
class InventoryFailed extends S.TaggedErrorClass<InventoryFailed>()(
|
||||||
|
"InventoryFailed",
|
||||||
|
{ message: S.String },
|
||||||
|
) {}
|
||||||
|
|
||||||
function extensionOf(path: string): string {
|
function extensionOf(path: string): string {
|
||||||
const match = path.match(/\.[cm]?tsx?$/);
|
const match = path.match(/\.[cm]?tsx?$/);
|
||||||
return match?.[0] ?? "";
|
return match?.[0] ?? "";
|
||||||
|
|
@ -42,30 +48,32 @@ function isSourceFile(path: string): boolean {
|
||||||
return sourceExtensions.has(extensionOf(path));
|
return sourceExtensions.has(extensionOf(path));
|
||||||
}
|
}
|
||||||
|
|
||||||
function walk(dir: string): string[] {
|
const walk = Effect.fn("inventory.walk")(function*(
|
||||||
if (!existsSync(dir)) {
|
dir: string,
|
||||||
return [];
|
): Effect.Effect<string[], PlatformError, FileSystem.FileSystem | Path.Path> {
|
||||||
}
|
const fs = yield* FileSystem.FileSystem;
|
||||||
|
const platformPath = yield* Path.Path;
|
||||||
|
const exists = yield* fs.exists(dir);
|
||||||
|
if (!exists) return [];
|
||||||
|
|
||||||
const files: string[] = [];
|
const entries = yield* fs.readDirectory(dir);
|
||||||
for (const entry of readdirSync(dir)) {
|
const chunks = yield* Effect.all(
|
||||||
if (entry === "dist" || entry === "node_modules" || entry === ".turbo") {
|
entries.map((entry) =>
|
||||||
continue;
|
Effect.gen(function*() {
|
||||||
}
|
if (ignoredDirectories.has(entry)) return [];
|
||||||
|
const fullPath = platformPath.join(dir, entry);
|
||||||
const path = join(dir, entry);
|
const stat = yield* fs.stat(fullPath);
|
||||||
const stat = statSync(path);
|
if (stat.type === "Directory") return yield* walk(fullPath);
|
||||||
if (stat.isDirectory()) {
|
return stat.type === "File" && isSourceFile(fullPath) ? [fullPath] : [];
|
||||||
files.push(...walk(path));
|
})
|
||||||
} else if (stat.isFile() && isSourceFile(path)) {
|
),
|
||||||
files.push(path);
|
{ concurrency: 16 },
|
||||||
}
|
);
|
||||||
}
|
return chunks.flat();
|
||||||
return files;
|
});
|
||||||
}
|
|
||||||
|
|
||||||
function isProductionPackageSource(path: string): boolean {
|
function isProductionPackageSource(path: string): boolean {
|
||||||
const rel = relative(root, path).split(sep).join("/");
|
const rel = path;
|
||||||
return (
|
return (
|
||||||
rel.startsWith("packages/") &&
|
rel.startsWith("packages/") &&
|
||||||
rel.includes("/src/") &&
|
rel.includes("/src/") &&
|
||||||
|
|
@ -107,8 +115,12 @@ function classify(scope: Scope, extendsText?: string): Pick<ClassFinding, "class
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function inspectFile(path: string): ClassFinding[] {
|
const inspectFile = Effect.fn("inventory.inspectFile")(function*(
|
||||||
const sourceText = readFileSync(path, "utf8");
|
path: string,
|
||||||
|
relativePath: string,
|
||||||
|
): Effect.Effect<ClassFinding[], PlatformError, FileSystem.FileSystem> {
|
||||||
|
const fs = yield* FileSystem.FileSystem;
|
||||||
|
const sourceText = yield* fs.readFileString(path);
|
||||||
const source = ts.createSourceFile(
|
const source = ts.createSourceFile(
|
||||||
path,
|
path,
|
||||||
sourceText,
|
sourceText,
|
||||||
|
|
@ -116,7 +128,7 @@ function inspectFile(path: string): ClassFinding[] {
|
||||||
true,
|
true,
|
||||||
path.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS,
|
path.endsWith(".tsx") ? ts.ScriptKind.TSX : ts.ScriptKind.TS,
|
||||||
);
|
);
|
||||||
const scope: Scope = isProductionPackageSource(path) ? "production" : "non-production";
|
const scope: Scope = isProductionPackageSource(relativePath) ? "production" : "non-production";
|
||||||
const findings: ClassFinding[] = [];
|
const findings: ClassFinding[] = [];
|
||||||
|
|
||||||
function visit(node: ts.Node): void {
|
function visit(node: ts.Node): void {
|
||||||
|
|
@ -125,7 +137,7 @@ function inspectFile(path: string): ClassFinding[] {
|
||||||
const extendsText = getExtendsText(node, source);
|
const extendsText = getExtendsText(node, source);
|
||||||
const { classification, reason } = classify(scope, extendsText);
|
const { classification, reason } = classify(scope, extendsText);
|
||||||
findings.push({
|
findings.push({
|
||||||
file: relative(root, path).split(sep).join("/"),
|
file: relativePath,
|
||||||
line: position.line + 1,
|
line: position.line + 1,
|
||||||
column: position.character + 1,
|
column: position.character + 1,
|
||||||
name: getClassName(node),
|
name: getClassName(node),
|
||||||
|
|
@ -141,14 +153,7 @@ function inspectFile(path: string): ClassFinding[] {
|
||||||
|
|
||||||
visit(source);
|
visit(source);
|
||||||
return findings;
|
return findings;
|
||||||
}
|
});
|
||||||
|
|
||||||
const files = [...walk(packagesSrc), ...walk(scriptsDir)].sort();
|
|
||||||
const findings = files.flatMap(inspectFile);
|
|
||||||
const productionFindings = findings.filter((finding) => finding.scope === "production");
|
|
||||||
const blocking = productionFindings.filter((finding) => finding.classification === "blocking");
|
|
||||||
const candidates = productionFindings.filter((finding) => finding.classification === "candidate-effect-exemption");
|
|
||||||
const nonProduction = findings.filter((finding) => finding.scope === "non-production");
|
|
||||||
|
|
||||||
function printGroup(title: string, group: ClassFinding[]): void {
|
function printGroup(title: string, group: ClassFinding[]): void {
|
||||||
console.log(`${title}: ${group.length}`);
|
console.log(`${title}: ${group.length}`);
|
||||||
|
|
@ -160,13 +165,37 @@ function printGroup(title: string, group: ClassFinding[]): void {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
printGroup("Blocking production native classes", blocking);
|
const program = Effect.fn("inventory.main")(function*() {
|
||||||
printGroup("Candidate Effect class-shaped exemptions", candidates);
|
const platformPath = yield* Path.Path;
|
||||||
printGroup("Non-production class declarations", nonProduction);
|
const packageFiles = yield* walk(platformPath.join(root, "packages"));
|
||||||
|
const scriptFiles = yield* walk(platformPath.join(root, "scripts"));
|
||||||
|
const files = A.sort([...packageFiles, ...scriptFiles], Order.String);
|
||||||
|
const findings = (yield* Effect.all(
|
||||||
|
files.map((file) =>
|
||||||
|
inspectFile(file, platformPath.relative(root, file).split(platformPath.sep).join("/"))
|
||||||
|
),
|
||||||
|
{ concurrency: 16 },
|
||||||
|
)).flat();
|
||||||
|
const productionFindings = findings.filter((finding) => finding.scope === "production");
|
||||||
|
const blocking = productionFindings.filter((finding) => finding.classification === "blocking");
|
||||||
|
const candidates = productionFindings.filter((finding) => finding.classification === "candidate-effect-exemption");
|
||||||
|
const nonProduction = findings.filter((finding) => finding.scope === "non-production");
|
||||||
|
|
||||||
if (blocking.length > 0) {
|
printGroup("Blocking production native classes", blocking);
|
||||||
console.error(`\nFound ${blocking.length} blocking production native class declarations.`);
|
printGroup("Candidate Effect class-shaped exemptions", candidates);
|
||||||
process.exit(1);
|
printGroup("Non-production class declarations", nonProduction);
|
||||||
}
|
|
||||||
|
|
||||||
console.log("\nNo blocking production native class declarations found.");
|
if (blocking.length > 0) {
|
||||||
|
const message = `Found ${blocking.length} blocking production native class declarations.`;
|
||||||
|
console.error(`\n${message}`);
|
||||||
|
return yield* InventoryFailed.make({ message });
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("\nNo blocking production native class declarations found.");
|
||||||
|
});
|
||||||
|
|
||||||
|
BunRuntime.runMain(
|
||||||
|
program().pipe(
|
||||||
|
Effect.provide(Layer.merge(BunFileSystem.layer, Path.layer)),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
|
||||||
|
|
@ -6,25 +6,117 @@
|
||||||
* Requires: gateway + config service running
|
* Requires: gateway + config service running
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const GATEWAY_URL = process.env.GATEWAY_URL ?? "http://localhost:8088";
|
import { BunRuntime } from "@effect/platform-bun";
|
||||||
|
import * as BunHttpClient from "@effect/platform-bun/BunHttpClient";
|
||||||
|
import { Config, Effect, Option as O, Schema as S } from "effect";
|
||||||
|
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
|
||||||
|
|
||||||
async function pushConfig(keys: string[], values: Record<string, unknown>): Promise<void> {
|
const DEFAULT_GATEWAY_URL = "http://localhost:8088";
|
||||||
const res = await fetch(`${GATEWAY_URL}/api/v1/config`, {
|
|
||||||
method: "POST",
|
class SeedConfigError extends S.TaggedErrorClass<SeedConfigError>()(
|
||||||
headers: { "Content-Type": "application/json" },
|
"SeedConfigError",
|
||||||
body: JSON.stringify({ operation: "put", keys, values }),
|
{
|
||||||
});
|
operation: S.String,
|
||||||
const data = await res.json();
|
message: S.String,
|
||||||
if (data.error) throw new Error(`Config push failed: ${data.error.message}`);
|
},
|
||||||
console.log(` Pushed config [${keys.join("/")}] → version ${data.version}`);
|
) {}
|
||||||
}
|
|
||||||
|
const GatewayErrorBody = S.Struct({
|
||||||
|
message: S.optionalKey(S.String),
|
||||||
|
});
|
||||||
|
|
||||||
|
const ConfigPushResponse = S.Struct({
|
||||||
|
version: S.optionalKey(S.Number),
|
||||||
|
error: S.optionalKey(GatewayErrorBody),
|
||||||
|
});
|
||||||
|
|
||||||
|
const stringifyJson = (operation: string, value: unknown) =>
|
||||||
|
S.encodeUnknownEffect(S.UnknownFromJsonString)(value).pipe(
|
||||||
|
Effect.mapError((cause) =>
|
||||||
|
SeedConfigError.make({
|
||||||
|
operation,
|
||||||
|
message: String(cause),
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
const decodeConfigResponse = (operation: string, value: unknown) =>
|
||||||
|
S.decodeUnknownEffect(ConfigPushResponse)(value).pipe(
|
||||||
|
Effect.mapError((cause) =>
|
||||||
|
SeedConfigError.make({
|
||||||
|
operation,
|
||||||
|
message: String(cause),
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
const postJson = Effect.fn("seed-config.postJson")(function* (
|
||||||
|
gatewayUrl: string,
|
||||||
|
path: string,
|
||||||
|
body: unknown,
|
||||||
|
) {
|
||||||
|
const bodyText = yield* stringifyJson("encode-request", body);
|
||||||
|
const request = HttpClientRequest.post(`${gatewayUrl}${path}`, { acceptJson: true }).pipe(
|
||||||
|
HttpClientRequest.bodyText(bodyText, "application/json"),
|
||||||
|
);
|
||||||
|
const response = yield* HttpClient.execute(request).pipe(
|
||||||
|
Effect.flatMap(HttpClientResponse.filterStatusOk),
|
||||||
|
Effect.mapError((cause) =>
|
||||||
|
SeedConfigError.make({
|
||||||
|
operation: "http-request",
|
||||||
|
message: String(cause),
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const responseText = yield* response.text.pipe(
|
||||||
|
Effect.mapError((cause) =>
|
||||||
|
SeedConfigError.make({
|
||||||
|
operation: "read-response",
|
||||||
|
message: String(cause),
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return yield* S.decodeUnknownEffect(S.UnknownFromJsonString)(responseText).pipe(
|
||||||
|
Effect.mapError((cause) =>
|
||||||
|
SeedConfigError.make({
|
||||||
|
operation: "decode-response-json",
|
||||||
|
message: String(cause),
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
const pushConfig = Effect.fn("seed-config.pushConfig")(function* (
|
||||||
|
gatewayUrl: string,
|
||||||
|
keys: ReadonlyArray<string>,
|
||||||
|
values: Record<string, unknown>,
|
||||||
|
) {
|
||||||
|
const data = yield* postJson(gatewayUrl, "/api/v1/config", {
|
||||||
|
operation: "put",
|
||||||
|
keys,
|
||||||
|
values,
|
||||||
|
}).pipe(Effect.flatMap((response) => decodeConfigResponse("decode-config-response", response)));
|
||||||
|
|
||||||
|
if (data.error !== undefined) {
|
||||||
|
return yield* SeedConfigError.make({
|
||||||
|
operation: "config-push",
|
||||||
|
message: data.error.message ?? "unknown gateway error",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(` Pushed config [${keys.join("/")}] → version ${data.version ?? "unknown"}`);
|
||||||
|
});
|
||||||
|
|
||||||
|
const main = Effect.fn("seed-config.main")(function* () {
|
||||||
|
const gatewayUrl = yield* Config.string("GATEWAY_URL").pipe(Config.withDefault(DEFAULT_GATEWAY_URL));
|
||||||
|
const braveApiKey = yield* Config.redacted("BRAVE_API_KEY").pipe(Config.option);
|
||||||
|
const hasBraveApiKey = O.isSome(braveApiKey);
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
|
||||||
console.log("Seeding TrustGraph configuration...\n");
|
console.log("Seeding TrustGraph configuration...\n");
|
||||||
|
|
||||||
// 1. Prompt templates
|
// 1. Prompt templates
|
||||||
console.log("── Prompt Templates ──");
|
console.log("── Prompt Templates ──");
|
||||||
await pushConfig(["prompt"], {
|
yield* pushConfig(gatewayUrl, ["prompt"], {
|
||||||
"extract-relationships": {
|
"extract-relationships": {
|
||||||
system: "You are a helpful assistant that extracts structured knowledge from text.",
|
system: "You are a helpful assistant that extracts structured knowledge from text.",
|
||||||
prompt: [
|
prompt: [
|
||||||
|
|
@ -142,7 +234,7 @@ async function main(): Promise<void> {
|
||||||
|
|
||||||
// 2. Flow definitions (default flow with all topic mappings)
|
// 2. Flow definitions (default flow with all topic mappings)
|
||||||
console.log("\n── Flow Definitions ──");
|
console.log("\n── Flow Definitions ──");
|
||||||
await pushConfig(["flows"], {
|
yield* pushConfig(gatewayUrl, ["flows"], {
|
||||||
default: {
|
default: {
|
||||||
topics: {
|
topics: {
|
||||||
// Document processing pipeline
|
// Document processing pipeline
|
||||||
|
|
@ -197,10 +289,9 @@ async function main(): Promise<void> {
|
||||||
|
|
||||||
// 3. MCP server configuration (external tool providers)
|
// 3. MCP server configuration (external tool providers)
|
||||||
console.log("\n── MCP Configuration ──");
|
console.log("\n── MCP Configuration ──");
|
||||||
const braveApiKey = process.env.BRAVE_API_KEY;
|
if (hasBraveApiKey) {
|
||||||
if (braveApiKey) {
|
yield* pushConfig(gatewayUrl, ["mcp"], {
|
||||||
await pushConfig(["mcp"], {
|
"brave-search": yield* stringifyJson("encode-brave-search-mcp", {
|
||||||
"brave-search": JSON.stringify({
|
|
||||||
url: "http://localhost:8383/mcp",
|
url: "http://localhost:8383/mcp",
|
||||||
"remote-name": "brave_web_search",
|
"remote-name": "brave_web_search",
|
||||||
}),
|
}),
|
||||||
|
|
@ -213,19 +304,19 @@ async function main(): Promise<void> {
|
||||||
// 4. Agent tool configuration (maps tools to implementations)
|
// 4. Agent tool configuration (maps tools to implementations)
|
||||||
console.log("\n── Tool Configuration ──");
|
console.log("\n── Tool Configuration ──");
|
||||||
const toolConfig: Record<string, string> = {
|
const toolConfig: Record<string, string> = {
|
||||||
"knowledge-query": JSON.stringify({
|
"knowledge-query": yield* stringifyJson("encode-knowledge-query-tool", {
|
||||||
type: "knowledge-query",
|
type: "knowledge-query",
|
||||||
name: "KnowledgeQuery",
|
name: "KnowledgeQuery",
|
||||||
description: "Query the knowledge graph for information about entities and their relationships.",
|
description: "Query the knowledge graph for information about entities and their relationships.",
|
||||||
group: ["default"],
|
group: ["default"],
|
||||||
}),
|
}),
|
||||||
"document-query": JSON.stringify({
|
"document-query": yield* stringifyJson("encode-document-query-tool", {
|
||||||
type: "document-query",
|
type: "document-query",
|
||||||
name: "DocumentQuery",
|
name: "DocumentQuery",
|
||||||
description: "Search the document library for relevant information using semantic search.",
|
description: "Search the document library for relevant information using semantic search.",
|
||||||
group: ["default"],
|
group: ["default"],
|
||||||
}),
|
}),
|
||||||
"triples-query": JSON.stringify({
|
"triples-query": yield* stringifyJson("encode-triples-query-tool", {
|
||||||
type: "triples-query",
|
type: "triples-query",
|
||||||
name: "TriplesQuery",
|
name: "TriplesQuery",
|
||||||
description: "Query for specific triples (subject-predicate-object relationships) in the knowledge graph.",
|
description: "Query for specific triples (subject-predicate-object relationships) in the knowledge graph.",
|
||||||
|
|
@ -234,8 +325,8 @@ async function main(): Promise<void> {
|
||||||
};
|
};
|
||||||
|
|
||||||
// Add Brave Search tool if API key is available
|
// Add Brave Search tool if API key is available
|
||||||
if (braveApiKey) {
|
if (hasBraveApiKey) {
|
||||||
toolConfig["brave-search"] = JSON.stringify({
|
toolConfig["brave-search"] = yield* stringifyJson("encode-brave-search-tool", {
|
||||||
type: "mcp-tool",
|
type: "mcp-tool",
|
||||||
name: "brave-search",
|
name: "brave-search",
|
||||||
description: "Search the web using Brave Search. Returns web search results including titles, URLs, and descriptions.",
|
description: "Search the web using Brave Search. Returns web search results including titles, URLs, and descriptions.",
|
||||||
|
|
@ -248,12 +339,9 @@ async function main(): Promise<void> {
|
||||||
console.log(" Brave Search tool added");
|
console.log(" Brave Search tool added");
|
||||||
}
|
}
|
||||||
|
|
||||||
await pushConfig(["tool"], toolConfig);
|
yield* pushConfig(gatewayUrl, ["tool"], toolConfig);
|
||||||
|
|
||||||
console.log("\nConfiguration seeded successfully.");
|
console.log("\nConfiguration seeded successfully.");
|
||||||
}
|
|
||||||
|
|
||||||
main().catch((err) => {
|
|
||||||
console.error("Seed failed:", err);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
BunRuntime.runMain(main().pipe(Effect.provide(BunHttpClient.layer)));
|
||||||
|
|
|
||||||
|
|
@ -16,17 +16,21 @@
|
||||||
* Also seeds config via the gateway if it's running.
|
* Also seeds config via the gateway if it's running.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { BunRuntime } from "@effect/platform-bun";
|
||||||
|
import * as BunHttpClient from "@effect/platform-bun/BunHttpClient";
|
||||||
|
import { Array as A, Config, Effect, Order, Schema as S } from "effect";
|
||||||
|
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
|
||||||
import { createClient, Graph } from "falkordb";
|
import { createClient, Graph } from "falkordb";
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Config
|
// Config
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
const FALKORDB_URL = process.env.FALKORDB_URL ?? "redis://localhost:6380";
|
const DEFAULT_FALKORDB_URL = "redis://localhost:6380";
|
||||||
const QDRANT_URL = process.env.QDRANT_URL ?? "http://localhost:6333";
|
const DEFAULT_QDRANT_URL = "http://localhost:6333";
|
||||||
const OLLAMA_URL = process.env.OLLAMA_URL ?? "http://localhost:11434";
|
const DEFAULT_OLLAMA_URL = "http://localhost:11434";
|
||||||
const GATEWAY_URL = process.env.GATEWAY_URL ?? "http://localhost:8088";
|
const DEFAULT_GATEWAY_URL = "http://localhost:8088";
|
||||||
const EMBED_MODEL = process.env.EMBED_MODEL ?? "mxbai-embed-large";
|
const DEFAULT_EMBED_MODEL = "mxbai-embed-large";
|
||||||
|
|
||||||
const USER = "default";
|
const USER = "default";
|
||||||
const COLLECTION = "default";
|
const COLLECTION = "default";
|
||||||
|
|
@ -44,6 +48,125 @@ interface RawTriple {
|
||||||
oIsEntity: boolean;
|
oIsEntity: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface DemoConfig {
|
||||||
|
readonly falkorDbUrl: string;
|
||||||
|
readonly qdrantUrl: string;
|
||||||
|
readonly ollamaUrl: string;
|
||||||
|
readonly gatewayUrl: string;
|
||||||
|
readonly embedModel: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
class SeedDemoError extends S.TaggedErrorClass<SeedDemoError>()(
|
||||||
|
"SeedDemoError",
|
||||||
|
{
|
||||||
|
operation: S.String,
|
||||||
|
message: S.String,
|
||||||
|
},
|
||||||
|
) {}
|
||||||
|
|
||||||
|
const GatewayErrorBody = S.Struct({
|
||||||
|
message: S.optionalKey(S.String),
|
||||||
|
});
|
||||||
|
|
||||||
|
const ConfigPushResponse = S.Struct({
|
||||||
|
version: S.optionalKey(S.Number),
|
||||||
|
error: S.optionalKey(GatewayErrorBody),
|
||||||
|
});
|
||||||
|
|
||||||
|
const OllamaEmbedResponse = S.Struct({
|
||||||
|
embeddings: S.Array(S.Array(S.Number)),
|
||||||
|
});
|
||||||
|
|
||||||
|
const loadConfig = Effect.fn("seed-demo.loadConfig")(function* () {
|
||||||
|
return {
|
||||||
|
falkorDbUrl: yield* Config.string("FALKORDB_URL").pipe(Config.withDefault(DEFAULT_FALKORDB_URL)),
|
||||||
|
qdrantUrl: yield* Config.string("QDRANT_URL").pipe(Config.withDefault(DEFAULT_QDRANT_URL)),
|
||||||
|
ollamaUrl: yield* Config.string("OLLAMA_URL").pipe(Config.withDefault(DEFAULT_OLLAMA_URL)),
|
||||||
|
gatewayUrl: yield* Config.string("GATEWAY_URL").pipe(Config.withDefault(DEFAULT_GATEWAY_URL)),
|
||||||
|
embedModel: yield* Config.string("EMBED_MODEL").pipe(Config.withDefault(DEFAULT_EMBED_MODEL)),
|
||||||
|
} satisfies DemoConfig;
|
||||||
|
});
|
||||||
|
|
||||||
|
const scriptError = (operation: string, cause: unknown) =>
|
||||||
|
SeedDemoError.make({
|
||||||
|
operation,
|
||||||
|
message: String(cause),
|
||||||
|
});
|
||||||
|
|
||||||
|
const stringifyJson = (operation: string, value: unknown) =>
|
||||||
|
S.encodeUnknownEffect(S.UnknownFromJsonString)(value).pipe(
|
||||||
|
Effect.mapError((cause) => scriptError(operation, cause)),
|
||||||
|
);
|
||||||
|
|
||||||
|
const decodeJsonText = (operation: string, value: string) =>
|
||||||
|
S.decodeUnknownEffect(S.UnknownFromJsonString)(value).pipe(
|
||||||
|
Effect.mapError((cause) => scriptError(operation, cause)),
|
||||||
|
);
|
||||||
|
|
||||||
|
const decodeWith = <A, I, R>(operation: string, schema: S.Codec<A, I, R>) => (value: unknown) =>
|
||||||
|
S.decodeUnknownEffect(schema)(value).pipe(
|
||||||
|
Effect.mapError((cause) => scriptError(operation, cause)),
|
||||||
|
);
|
||||||
|
|
||||||
|
const readResponseText = Effect.fn("seed-demo.readResponseText")(function* (
|
||||||
|
operation: string,
|
||||||
|
response: HttpClientResponse.HttpClientResponse,
|
||||||
|
) {
|
||||||
|
return yield* response.text.pipe(
|
||||||
|
Effect.mapError((cause) => scriptError(`${operation}.read-response`, cause)),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
const executeOkText = Effect.fn("seed-demo.executeOkText")(function* (
|
||||||
|
operation: string,
|
||||||
|
request: HttpClientRequest.HttpClientRequest,
|
||||||
|
) {
|
||||||
|
const response = yield* HttpClient.execute(request).pipe(
|
||||||
|
Effect.flatMap(HttpClientResponse.filterStatusOk),
|
||||||
|
Effect.mapError((cause) => scriptError(`${operation}.http`, cause)),
|
||||||
|
);
|
||||||
|
return yield* readResponseText(operation, response);
|
||||||
|
});
|
||||||
|
|
||||||
|
const postJsonText = Effect.fn("seed-demo.postJsonText")(function* (
|
||||||
|
operation: string,
|
||||||
|
url: string,
|
||||||
|
body: unknown,
|
||||||
|
) {
|
||||||
|
const bodyText = yield* stringifyJson(`${operation}.encode-request`, body);
|
||||||
|
const request = HttpClientRequest.post(url, { acceptJson: true }).pipe(
|
||||||
|
HttpClientRequest.bodyText(bodyText, "application/json"),
|
||||||
|
);
|
||||||
|
return yield* executeOkText(operation, request);
|
||||||
|
});
|
||||||
|
|
||||||
|
const putJsonText = Effect.fn("seed-demo.putJsonText")(function* (
|
||||||
|
operation: string,
|
||||||
|
url: string,
|
||||||
|
body: unknown,
|
||||||
|
) {
|
||||||
|
const bodyText = yield* stringifyJson(`${operation}.encode-request`, body);
|
||||||
|
const request = HttpClientRequest.put(url, { acceptJson: true }).pipe(
|
||||||
|
HttpClientRequest.bodyText(bodyText, "application/json"),
|
||||||
|
);
|
||||||
|
return yield* executeOkText(operation, request);
|
||||||
|
});
|
||||||
|
|
||||||
|
const httpAvailable = Effect.fn("seed-demo.httpAvailable")(function* (url: string) {
|
||||||
|
return yield* HttpClient.get(url).pipe(
|
||||||
|
Effect.timeout("3 seconds"),
|
||||||
|
Effect.map((response) => response.status >= 200 && response.status < 300),
|
||||||
|
Effect.catch(() => Effect.succeed(false)),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
const resourceExists = Effect.fn("seed-demo.resourceExists")(function* (url: string) {
|
||||||
|
return yield* HttpClient.get(url).pipe(
|
||||||
|
Effect.map((response) => response.status >= 200 && response.status < 300),
|
||||||
|
Effect.catch(() => Effect.succeed(false)),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Demo Knowledge Graph — AI Industry
|
// Demo Knowledge Graph — AI Industry
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
@ -473,16 +596,16 @@ function collectEntities(triples: RawTriple[]): string[] {
|
||||||
entities.add(t.o);
|
entities.add(t.o);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return [...entities].sort();
|
return A.sort(Array.from(entities), Order.String);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Connectivity checks
|
// Connectivity checks
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function checkFalkorDB(): Promise<boolean> {
|
async function checkFalkorDB(config: DemoConfig): Promise<boolean> {
|
||||||
try {
|
try {
|
||||||
const client = createClient({ url: FALKORDB_URL });
|
const client = createClient({ url: config.falkorDbUrl });
|
||||||
await client.connect();
|
await client.connect();
|
||||||
await client.ping();
|
await client.ping();
|
||||||
await client.disconnect();
|
await client.disconnect();
|
||||||
|
|
@ -492,39 +615,18 @@ async function checkFalkorDB(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function checkQdrant(): Promise<boolean> {
|
const checkQdrant = (config: DemoConfig) => httpAvailable(`${config.qdrantUrl}/collections`);
|
||||||
try {
|
|
||||||
const res = await fetch(`${QDRANT_URL}/collections`, { signal: AbortSignal.timeout(3000) });
|
|
||||||
return res.ok;
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function checkOllama(): Promise<boolean> {
|
const checkOllama = (config: DemoConfig) => httpAvailable(`${config.ollamaUrl}/api/tags`);
|
||||||
try {
|
|
||||||
const res = await fetch(`${OLLAMA_URL}/api/tags`, { signal: AbortSignal.timeout(3000) });
|
|
||||||
return res.ok;
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function checkGateway(): Promise<boolean> {
|
const checkGateway = (config: DemoConfig) => httpAvailable(`${config.gatewayUrl}/api/v1/metrics`);
|
||||||
try {
|
|
||||||
const res = await fetch(`${GATEWAY_URL}/api/v1/metrics`, { signal: AbortSignal.timeout(3000) });
|
|
||||||
return res.ok;
|
|
||||||
} catch {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// FalkorDB seeding
|
// FalkorDB seeding
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function seedFalkorDB(triples: RawTriple[]): Promise<void> {
|
async function seedFalkorDB(config: DemoConfig, triples: RawTriple[]): Promise<void> {
|
||||||
const client = createClient({ url: FALKORDB_URL });
|
const client = createClient({ url: config.falkorDbUrl });
|
||||||
await client.connect();
|
await client.connect();
|
||||||
const graph = new Graph(client, DATABASE);
|
const graph = new Graph(client, DATABASE);
|
||||||
|
|
||||||
|
|
@ -581,19 +683,16 @@ async function seedFalkorDB(triples: RawTriple[]): Promise<void> {
|
||||||
// Ollama embeddings
|
// Ollama embeddings
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function embed(texts: string[]): Promise<number[][]> {
|
const embed = Effect.fn("seed-demo.embed")(function* (config: DemoConfig, texts: string[]) {
|
||||||
const res = await fetch(`${OLLAMA_URL}/api/embed`, {
|
const responseText = yield* postJsonText("ollama.embed", `${config.ollamaUrl}/api/embed`, {
|
||||||
method: "POST",
|
model: config.embedModel,
|
||||||
headers: { "Content-Type": "application/json" },
|
input: texts,
|
||||||
body: JSON.stringify({ model: EMBED_MODEL, input: texts }),
|
|
||||||
});
|
});
|
||||||
if (!res.ok) {
|
const data = yield* decodeJsonText("ollama.embed.decode-json", responseText).pipe(
|
||||||
const body = await res.text();
|
Effect.flatMap(decodeWith("ollama.embed.decode-response", OllamaEmbedResponse)),
|
||||||
throw new Error(`Ollama embed failed (${res.status}): ${body}`);
|
);
|
||||||
}
|
|
||||||
const data = (await res.json()) as { embeddings: number[][] };
|
|
||||||
return data.embeddings;
|
return data.embeddings;
|
||||||
}
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Document chunks for Doc RAG
|
// Document chunks for Doc RAG
|
||||||
|
|
@ -705,7 +804,7 @@ const DOCUMENT_CHUNKS: Array<{ id: string; content: string }> = [
|
||||||
// Qdrant seeding (document embeddings)
|
// Qdrant seeding (document embeddings)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function seedDocumentChunks(): Promise<void> {
|
const seedDocumentChunks = Effect.fn("seed-demo.seedDocumentChunks")(function* (config: DemoConfig) {
|
||||||
// Embed all chunk content
|
// Embed all chunk content
|
||||||
const BATCH_SIZE = 32;
|
const BATCH_SIZE = 32;
|
||||||
const allVectors: number[][] = [];
|
const allVectors: number[][] = [];
|
||||||
|
|
@ -713,7 +812,7 @@ async function seedDocumentChunks(): Promise<void> {
|
||||||
|
|
||||||
for (let i = 0; i < texts.length; i += BATCH_SIZE) {
|
for (let i = 0; i < texts.length; i += BATCH_SIZE) {
|
||||||
const batch = texts.slice(i, i + BATCH_SIZE);
|
const batch = texts.slice(i, i + BATCH_SIZE);
|
||||||
const vecs = await embed(batch);
|
const vecs = yield* embed(config, batch);
|
||||||
allVectors.push(...vecs);
|
allVectors.push(...vecs);
|
||||||
process.stdout.write(
|
process.stdout.write(
|
||||||
`\r Embedding doc chunks: ${Math.min(i + BATCH_SIZE, texts.length)}/${texts.length}`,
|
`\r Embedding doc chunks: ${Math.min(i + BATCH_SIZE, texts.length)}/${texts.length}`,
|
||||||
|
|
@ -725,14 +824,10 @@ async function seedDocumentChunks(): Promise<void> {
|
||||||
const collectionName = `d_${USER}_${COLLECTION}_${dim}`;
|
const collectionName = `d_${USER}_${COLLECTION}_${dim}`;
|
||||||
|
|
||||||
// Create collection if needed
|
// Create collection if needed
|
||||||
const existsRes = await fetch(`${QDRANT_URL}/collections/${collectionName}`);
|
const exists = yield* resourceExists(`${config.qdrantUrl}/collections/${collectionName}`);
|
||||||
if (!existsRes.ok) {
|
if (!exists) {
|
||||||
await fetch(`${QDRANT_URL}/collections/${collectionName}`, {
|
yield* putJsonText("qdrant.create-doc-collection", `${config.qdrantUrl}/collections/${collectionName}`, {
|
||||||
method: "PUT",
|
vectors: { size: dim, distance: "Cosine" },
|
||||||
headers: { "Content-Type": "application/json" },
|
|
||||||
body: JSON.stringify({
|
|
||||||
vectors: { size: dim, distance: "Cosine" },
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
console.log(` Created Qdrant collection: ${collectionName} (dim=${dim})`);
|
console.log(` Created Qdrant collection: ${collectionName} (dim=${dim})`);
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -749,32 +844,28 @@ async function seedDocumentChunks(): Promise<void> {
|
||||||
},
|
},
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const res = await fetch(`${QDRANT_URL}/collections/${collectionName}/points`, {
|
yield* putJsonText("qdrant.upsert-doc-points", `${config.qdrantUrl}/collections/${collectionName}/points`, {
|
||||||
method: "PUT",
|
points,
|
||||||
headers: { "Content-Type": "application/json" },
|
|
||||||
body: JSON.stringify({ points }),
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
const body = await res.text();
|
|
||||||
throw new Error(`Qdrant doc upsert failed: ${body}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(` Qdrant: ${points.length} document chunk embeddings stored in ${collectionName}`);
|
console.log(` Qdrant: ${points.length} document chunk embeddings stored in ${collectionName}`);
|
||||||
}
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Qdrant seeding (graph embeddings)
|
// Qdrant seeding (graph embeddings)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function seedQdrant(entities: string[]): Promise<void> {
|
const seedQdrant = Effect.fn("seed-demo.seedQdrant")(function* (
|
||||||
|
config: DemoConfig,
|
||||||
|
entities: string[],
|
||||||
|
) {
|
||||||
// Batch embed in groups of 32
|
// Batch embed in groups of 32
|
||||||
const BATCH_SIZE = 32;
|
const BATCH_SIZE = 32;
|
||||||
const allVectors: number[][] = [];
|
const allVectors: number[][] = [];
|
||||||
|
|
||||||
for (let i = 0; i < entities.length; i += BATCH_SIZE) {
|
for (let i = 0; i < entities.length; i += BATCH_SIZE) {
|
||||||
const batch = entities.slice(i, i + BATCH_SIZE);
|
const batch = entities.slice(i, i + BATCH_SIZE);
|
||||||
const vecs = await embed(batch);
|
const vecs = yield* embed(config, batch);
|
||||||
allVectors.push(...vecs);
|
allVectors.push(...vecs);
|
||||||
process.stdout.write(
|
process.stdout.write(
|
||||||
`\r Embedding entities: ${Math.min(i + BATCH_SIZE, entities.length)}/${entities.length}`,
|
`\r Embedding entities: ${Math.min(i + BATCH_SIZE, entities.length)}/${entities.length}`,
|
||||||
|
|
@ -786,14 +877,10 @@ async function seedQdrant(entities: string[]): Promise<void> {
|
||||||
const collectionName = `t_${USER}_${COLLECTION}_${dim}`;
|
const collectionName = `t_${USER}_${COLLECTION}_${dim}`;
|
||||||
|
|
||||||
// Create collection if needed
|
// Create collection if needed
|
||||||
const existsRes = await fetch(`${QDRANT_URL}/collections/${collectionName}`);
|
const exists = yield* resourceExists(`${config.qdrantUrl}/collections/${collectionName}`);
|
||||||
if (!existsRes.ok) {
|
if (!exists) {
|
||||||
await fetch(`${QDRANT_URL}/collections/${collectionName}`, {
|
yield* putJsonText("qdrant.create-entity-collection", `${config.qdrantUrl}/collections/${collectionName}`, {
|
||||||
method: "PUT",
|
vectors: { size: dim, distance: "Cosine" },
|
||||||
headers: { "Content-Type": "application/json" },
|
|
||||||
body: JSON.stringify({
|
|
||||||
vectors: { size: dim, distance: "Cosine" },
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
console.log(` Created Qdrant collection: ${collectionName} (dim=${dim})`);
|
console.log(` Created Qdrant collection: ${collectionName} (dim=${dim})`);
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -811,41 +898,44 @@ async function seedQdrant(entities: string[]): Promise<void> {
|
||||||
payload: { entity },
|
payload: { entity },
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const res = await fetch(`${QDRANT_URL}/collections/${collectionName}/points`, {
|
yield* putJsonText("qdrant.upsert-entity-points", `${config.qdrantUrl}/collections/${collectionName}/points`, {
|
||||||
method: "PUT",
|
points,
|
||||||
headers: { "Content-Type": "application/json" },
|
|
||||||
body: JSON.stringify({ points }),
|
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
const body = await res.text();
|
|
||||||
throw new Error(`Qdrant upsert failed: ${body}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
upserted += points.length;
|
upserted += points.length;
|
||||||
process.stdout.write(`\r Upserting to Qdrant: ${upserted}/${entities.length}`);
|
process.stdout.write(`\r Upserting to Qdrant: ${upserted}/${entities.length}`);
|
||||||
}
|
}
|
||||||
console.log();
|
console.log();
|
||||||
console.log(` Qdrant: ${upserted} entity embeddings stored`);
|
console.log(` Qdrant: ${upserted} entity embeddings stored`);
|
||||||
}
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Config seeding (via gateway)
|
// Config seeding (via gateway)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function seedConfig(): Promise<void> {
|
const seedConfig = Effect.fn("seed-demo.seedConfig")(function* (config: DemoConfig) {
|
||||||
async function pushConfig(keys: string[], values: Record<string, unknown>): Promise<void> {
|
const pushConfig = Effect.fn("seed-demo.seedConfig.pushConfig")(function* (
|
||||||
const res = await fetch(`${GATEWAY_URL}/api/v1/config`, {
|
keys: ReadonlyArray<string>,
|
||||||
method: "POST",
|
values: Record<string, unknown>,
|
||||||
headers: { "Content-Type": "application/json" },
|
) {
|
||||||
body: JSON.stringify({ operation: "put", keys, values }),
|
const responseText = yield* postJsonText("config.put", `${config.gatewayUrl}/api/v1/config`, {
|
||||||
|
operation: "put",
|
||||||
|
keys,
|
||||||
|
values,
|
||||||
});
|
});
|
||||||
const data = (await res.json()) as { error?: { message: string }; version?: number };
|
const data = yield* decodeJsonText("config.put.decode-json", responseText).pipe(
|
||||||
if (data.error) throw new Error(`Config push failed: ${data.error.message}`);
|
Effect.flatMap(decodeWith("config.put.decode-response", ConfigPushResponse)),
|
||||||
console.log(` Config [${keys.join("/")}] → version ${data.version}`);
|
);
|
||||||
}
|
if (data.error !== undefined) {
|
||||||
|
return yield* SeedDemoError.make({
|
||||||
|
operation: "config.put",
|
||||||
|
message: data.error.message ?? "unknown gateway error",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
console.log(` Config [${keys.join("/")}] → version ${data.version ?? "unknown"}`);
|
||||||
|
});
|
||||||
|
|
||||||
await pushConfig(["prompt"], {
|
yield* pushConfig(["prompt"], {
|
||||||
"extract-relationships": {
|
"extract-relationships": {
|
||||||
system: "You are a helpful assistant that extracts structured knowledge from text.",
|
system: "You are a helpful assistant that extracts structured knowledge from text.",
|
||||||
prompt: [
|
prompt: [
|
||||||
|
|
@ -914,7 +1004,7 @@ async function seedConfig(): Promise<void> {
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
await pushConfig(["flows"], {
|
yield* pushConfig(["flows"], {
|
||||||
default: {
|
default: {
|
||||||
topics: {
|
topics: {
|
||||||
"decode-input": "tg.flow.document",
|
"decode-input": "tg.flow.document",
|
||||||
|
|
@ -951,36 +1041,49 @@ async function seedConfig(): Promise<void> {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
});
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Main
|
// Main
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
const main = Effect.fn("seed-demo.main")(function* () {
|
||||||
|
const config = yield* loadConfig();
|
||||||
|
|
||||||
console.log("╔══════════════════════════════════════════════════════════╗");
|
console.log("╔══════════════════════════════════════════════════════════╗");
|
||||||
console.log("║ TrustGraph Demo Seeder — AI Industry KG ║");
|
console.log("║ TrustGraph Demo Seeder — AI Industry KG ║");
|
||||||
console.log("╚══════════════════════════════════════════════════════════╝\n");
|
console.log("╚══════════════════════════════════════════════════════════╝\n");
|
||||||
|
|
||||||
// Check services
|
// Check services
|
||||||
const [hasFalkor, hasQdrant, hasOllama, hasGateway] = await Promise.all([
|
const availability = yield* Effect.all({
|
||||||
checkFalkorDB(),
|
falkor: Effect.tryPromise({
|
||||||
checkQdrant(),
|
try: () => checkFalkorDB(config),
|
||||||
checkOllama(),
|
catch: (cause) => scriptError("check-falkordb", cause),
|
||||||
checkGateway(),
|
}).pipe(Effect.catch(() => Effect.succeed(false))),
|
||||||
]);
|
qdrant: checkQdrant(config),
|
||||||
|
ollama: checkOllama(config),
|
||||||
|
gateway: checkGateway(config),
|
||||||
|
}, { concurrency: "unbounded" });
|
||||||
|
|
||||||
|
const hasFalkor = availability.falkor;
|
||||||
|
const hasQdrant = availability.qdrant;
|
||||||
|
const hasOllama = availability.ollama;
|
||||||
|
const hasGateway = availability.gateway;
|
||||||
|
|
||||||
console.log("Service availability:");
|
console.log("Service availability:");
|
||||||
console.log(` FalkorDB (${FALKORDB_URL}): ${hasFalkor ? "✓" : "✗"}`);
|
console.log(` FalkorDB (${config.falkorDbUrl}): ${hasFalkor ? "✓" : "✗"}`);
|
||||||
console.log(` Qdrant (${QDRANT_URL}): ${hasQdrant ? "✓" : "✗"}`);
|
console.log(` Qdrant (${config.qdrantUrl}): ${hasQdrant ? "✓" : "✗"}`);
|
||||||
console.log(` Ollama (${OLLAMA_URL}): ${hasOllama ? "✓" : "✗"}`);
|
console.log(` Ollama (${config.ollamaUrl}): ${hasOllama ? "✓" : "✗"}`);
|
||||||
console.log(` Gateway (${GATEWAY_URL}): ${hasGateway ? "✓" : "✗"}`);
|
console.log(` Gateway (${config.gatewayUrl}): ${hasGateway ? "✓" : "✗"}`);
|
||||||
console.log();
|
console.log();
|
||||||
|
|
||||||
if (!hasFalkor && !hasQdrant && !hasGateway) {
|
if (!hasFalkor && !hasQdrant && !hasGateway) {
|
||||||
console.error("No services available. Start the TrustGraph stack first:");
|
console.error("No services available. Start the TrustGraph stack first:");
|
||||||
console.error(" cd ts/deploy && docker compose up -d falkordb qdrant ollama nats");
|
console.error(" cd ts/deploy && docker compose up -d falkordb qdrant ollama nats");
|
||||||
process.exit(1);
|
return yield* SeedDemoError.make({
|
||||||
|
operation: "service-check",
|
||||||
|
message: "no seed targets available",
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const triples = buildTriples();
|
const triples = buildTriples();
|
||||||
|
|
@ -991,7 +1094,10 @@ async function main(): Promise<void> {
|
||||||
// Seed FalkorDB
|
// Seed FalkorDB
|
||||||
if (hasFalkor) {
|
if (hasFalkor) {
|
||||||
console.log("── Seeding FalkorDB ──");
|
console.log("── Seeding FalkorDB ──");
|
||||||
await seedFalkorDB(triples);
|
yield* Effect.tryPromise({
|
||||||
|
try: () => seedFalkorDB(config, triples),
|
||||||
|
catch: (cause) => scriptError("seed-falkordb", cause),
|
||||||
|
});
|
||||||
console.log();
|
console.log();
|
||||||
} else {
|
} else {
|
||||||
console.log("⚠ Skipping FalkorDB (not available)\n");
|
console.log("⚠ Skipping FalkorDB (not available)\n");
|
||||||
|
|
@ -1000,11 +1106,11 @@ async function main(): Promise<void> {
|
||||||
// Seed Qdrant (requires Ollama for embeddings)
|
// Seed Qdrant (requires Ollama for embeddings)
|
||||||
if (hasQdrant && hasOllama) {
|
if (hasQdrant && hasOllama) {
|
||||||
console.log("── Seeding Qdrant (entity embeddings) ──");
|
console.log("── Seeding Qdrant (entity embeddings) ──");
|
||||||
await seedQdrant(entities);
|
yield* seedQdrant(config, entities);
|
||||||
console.log();
|
console.log();
|
||||||
|
|
||||||
console.log("── Seeding Qdrant (document chunk embeddings) ──");
|
console.log("── Seeding Qdrant (document chunk embeddings) ──");
|
||||||
await seedDocumentChunks();
|
yield* seedDocumentChunks(config);
|
||||||
console.log();
|
console.log();
|
||||||
} else if (hasQdrant) {
|
} else if (hasQdrant) {
|
||||||
console.log("⚠ Skipping Qdrant embeddings (Ollama not available for embedding generation)\n");
|
console.log("⚠ Skipping Qdrant embeddings (Ollama not available for embedding generation)\n");
|
||||||
|
|
@ -1015,7 +1121,7 @@ async function main(): Promise<void> {
|
||||||
// Seed config via gateway
|
// Seed config via gateway
|
||||||
if (hasGateway) {
|
if (hasGateway) {
|
||||||
console.log("── Seeding Config (prompt templates + flows) ──");
|
console.log("── Seeding Config (prompt templates + flows) ──");
|
||||||
await seedConfig();
|
yield* seedConfig(config);
|
||||||
console.log();
|
console.log();
|
||||||
} else {
|
} else {
|
||||||
console.log("⚠ Skipping config (gateway not available — run `pnpm seed` separately)\n");
|
console.log("⚠ Skipping config (gateway not available — run `pnpm seed` separately)\n");
|
||||||
|
|
@ -1036,9 +1142,6 @@ async function main(): Promise<void> {
|
||||||
console.log(" • What is the Transformer architecture?");
|
console.log(" • What is the Transformer architecture?");
|
||||||
console.log(" • Tell me about Demis Hassabis and his achievements");
|
console.log(" • Tell me about Demis Hassabis and his achievements");
|
||||||
console.log("═══════════════════════════════════════════════════════════");
|
console.log("═══════════════════════════════════════════════════════════");
|
||||||
}
|
|
||||||
|
|
||||||
main().catch((err) => {
|
|
||||||
console.error("\nSeed failed:", err);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
BunRuntime.runMain(main().pipe(Effect.provide(BunHttpClient.layer)));
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,64 @@
|
||||||
* Requires: gateway + flow-manager + config service running
|
* Requires: gateway + flow-manager + config service running
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const GATEWAY_URL = process.env.GATEWAY_URL ?? "http://localhost:8088";
|
import { BunRuntime } from "@effect/platform-bun";
|
||||||
|
import * as BunHttpClient from "@effect/platform-bun/BunHttpClient";
|
||||||
|
import { Array as A, Config, Effect, Order, Schema as S } from "effect";
|
||||||
|
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
|
||||||
|
|
||||||
|
const DEFAULT_GATEWAY_URL = "http://localhost:8088";
|
||||||
|
|
||||||
|
class SeedFlowsError extends S.TaggedErrorClass<SeedFlowsError>()(
|
||||||
|
"SeedFlowsError",
|
||||||
|
{
|
||||||
|
operation: S.String,
|
||||||
|
message: S.String,
|
||||||
|
},
|
||||||
|
) {}
|
||||||
|
|
||||||
|
const GatewayErrorBody = S.Struct({
|
||||||
|
message: S.optionalKey(S.String),
|
||||||
|
});
|
||||||
|
|
||||||
|
const FlowListResponse = S.Struct({
|
||||||
|
"flow-ids": S.optionalKey(S.Array(S.String)),
|
||||||
|
error: S.optionalKey(GatewayErrorBody),
|
||||||
|
});
|
||||||
|
|
||||||
|
const GatewayResponse = S.Struct({
|
||||||
|
version: S.optionalKey(S.Number),
|
||||||
|
error: S.optionalKey(GatewayErrorBody),
|
||||||
|
});
|
||||||
|
|
||||||
|
const stringifyJson = (operation: string, value: unknown) =>
|
||||||
|
S.encodeUnknownEffect(S.UnknownFromJsonString)(value).pipe(
|
||||||
|
Effect.mapError((cause) =>
|
||||||
|
SeedFlowsError.make({
|
||||||
|
operation,
|
||||||
|
message: String(cause),
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
const decodeJsonText = (operation: string, value: string) =>
|
||||||
|
S.decodeUnknownEffect(S.UnknownFromJsonString)(value).pipe(
|
||||||
|
Effect.mapError((cause) =>
|
||||||
|
SeedFlowsError.make({
|
||||||
|
operation,
|
||||||
|
message: String(cause),
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
const decodeWith = <A, I, R>(operation: string, schema: S.Codec<A, I, R>) => (value: unknown) =>
|
||||||
|
S.decodeUnknownEffect(schema)(value).pipe(
|
||||||
|
Effect.mapError((cause) =>
|
||||||
|
SeedFlowsError.make({
|
||||||
|
operation,
|
||||||
|
message: String(cause),
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
const FLOW_TOPICS = {
|
const FLOW_TOPICS = {
|
||||||
// Document processing pipeline
|
// Document processing pipeline
|
||||||
|
|
@ -79,37 +136,66 @@ const SEEDED_FLOWS = [
|
||||||
},
|
},
|
||||||
] as const;
|
] as const;
|
||||||
|
|
||||||
async function postJson<T>(path: string, body: Record<string, unknown>): Promise<T> {
|
const postJson = Effect.fn("seed-flows.postJson")(function* (
|
||||||
const res = await fetch(`${GATEWAY_URL}${path}`, {
|
gatewayUrl: string,
|
||||||
method: "POST",
|
path: string,
|
||||||
headers: { "Content-Type": "application/json" },
|
body: Record<string, unknown>,
|
||||||
body: JSON.stringify(body),
|
) {
|
||||||
});
|
const bodyText = yield* stringifyJson("encode-request", body);
|
||||||
const data = await res.json() as T & { error?: { message?: string } };
|
const request = HttpClientRequest.post(`${gatewayUrl}${path}`, { acceptJson: true }).pipe(
|
||||||
if (!res.ok) {
|
HttpClientRequest.bodyText(bodyText, "application/json"),
|
||||||
throw new Error(`HTTP ${res.status}: ${JSON.stringify(data)}`);
|
);
|
||||||
}
|
const response = yield* HttpClient.execute(request).pipe(
|
||||||
|
Effect.flatMap(HttpClientResponse.filterStatusOk),
|
||||||
|
Effect.mapError((cause) =>
|
||||||
|
SeedFlowsError.make({
|
||||||
|
operation: "http-request",
|
||||||
|
message: String(cause),
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const responseText = yield* response.text.pipe(
|
||||||
|
Effect.mapError((cause) =>
|
||||||
|
SeedFlowsError.make({
|
||||||
|
operation: "read-response",
|
||||||
|
message: String(cause),
|
||||||
|
})
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return yield* decodeJsonText("decode-response-json", responseText);
|
||||||
|
});
|
||||||
|
|
||||||
|
const failOnGatewayError = Effect.fn("seed-flows.failOnGatewayError")(function* (
|
||||||
|
operation: string,
|
||||||
|
data: { readonly error?: { readonly message?: string } },
|
||||||
|
) {
|
||||||
if (data.error !== undefined) {
|
if (data.error !== undefined) {
|
||||||
throw new Error(data.error.message ?? JSON.stringify(data.error));
|
return yield* SeedFlowsError.make({
|
||||||
|
operation,
|
||||||
|
message: data.error.message ?? "unknown gateway error",
|
||||||
|
});
|
||||||
}
|
}
|
||||||
return data;
|
});
|
||||||
}
|
|
||||||
|
|
||||||
async function listFlows(): Promise<Set<string>> {
|
const listFlows = Effect.fn("seed-flows.listFlows")(function* (gatewayUrl: string) {
|
||||||
const response = await postJson<{ "flow-ids"?: string[] }>("/api/v1/flow", {
|
const response = yield* postJson(gatewayUrl, "/api/v1/flow", {
|
||||||
operation: "list-flows",
|
operation: "list-flows",
|
||||||
});
|
}).pipe(Effect.flatMap(decodeWith("decode-flow-list", FlowListResponse)));
|
||||||
|
yield* failOnGatewayError("list-flows", response);
|
||||||
return new Set(response["flow-ids"] ?? []);
|
return new Set(response["flow-ids"] ?? []);
|
||||||
}
|
});
|
||||||
|
|
||||||
async function startMissingFlows(existing: Set<string>): Promise<void> {
|
const startMissingFlows = Effect.fn("seed-flows.startMissingFlows")(function* (
|
||||||
|
gatewayUrl: string,
|
||||||
|
existing: Set<string>,
|
||||||
|
) {
|
||||||
for (const flow of SEEDED_FLOWS) {
|
for (const flow of SEEDED_FLOWS) {
|
||||||
if (existing.has(flow.id)) {
|
if (existing.has(flow.id)) {
|
||||||
console.log(` Flow ${flow.id}: already running`);
|
console.log(` Flow ${flow.id}: already running`);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
await postJson("/api/v1/flow", {
|
const response = yield* postJson(gatewayUrl, "/api/v1/flow", {
|
||||||
operation: "start-flow",
|
operation: "start-flow",
|
||||||
"flow-id": flow.id,
|
"flow-id": flow.id,
|
||||||
"blueprint-name": "default",
|
"blueprint-name": "default",
|
||||||
|
|
@ -118,40 +204,41 @@ async function startMissingFlows(existing: Set<string>): Promise<void> {
|
||||||
user: "default",
|
user: "default",
|
||||||
collection: "default",
|
collection: "default",
|
||||||
},
|
},
|
||||||
});
|
}).pipe(Effect.flatMap(decodeWith("decode-start-flow", GatewayResponse)));
|
||||||
|
yield* failOnGatewayError("start-flow", response);
|
||||||
existing.add(flow.id);
|
existing.add(flow.id);
|
||||||
console.log(` Flow ${flow.id}: started`);
|
console.log(` Flow ${flow.id}: started`);
|
||||||
}
|
}
|
||||||
}
|
});
|
||||||
|
|
||||||
async function ensureFlowConfig(): Promise<void> {
|
const ensureFlowConfig = Effect.fn("seed-flows.ensureFlowConfig")(function* (gatewayUrl: string) {
|
||||||
const values = Object.fromEntries(
|
const values = Object.fromEntries(
|
||||||
SEEDED_FLOWS.map((flow) => [flow.id, { topics: FLOW_TOPICS }]),
|
SEEDED_FLOWS.map((flow) => [flow.id, { topics: FLOW_TOPICS }]),
|
||||||
);
|
);
|
||||||
|
|
||||||
const response = await postJson<{ version?: number }>("/api/v1/config", {
|
const response = yield* postJson(gatewayUrl, "/api/v1/config", {
|
||||||
operation: "put",
|
operation: "put",
|
||||||
keys: ["flows"],
|
keys: ["flows"],
|
||||||
values,
|
values,
|
||||||
});
|
}).pipe(Effect.flatMap(decodeWith("decode-flow-config", GatewayResponse)));
|
||||||
|
yield* failOnGatewayError("flow-config", response);
|
||||||
console.log(` Flow config topics pushed -> version ${response.version ?? "unknown"}`);
|
console.log(` Flow config topics pushed -> version ${response.version ?? "unknown"}`);
|
||||||
}
|
});
|
||||||
|
|
||||||
|
const main = Effect.fn("seed-flows.main")(function* () {
|
||||||
|
const gatewayUrl = yield* Config.string("GATEWAY_URL").pipe(Config.withDefault(DEFAULT_GATEWAY_URL));
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
|
||||||
console.log("Seeding TrustGraph flows...\n");
|
console.log("Seeding TrustGraph flows...\n");
|
||||||
|
|
||||||
const existing = await listFlows();
|
const existing = yield* listFlows(gatewayUrl);
|
||||||
await startMissingFlows(existing);
|
yield* startMissingFlows(gatewayUrl, existing);
|
||||||
|
|
||||||
console.log("\nAligning flow config topics...");
|
console.log("\nAligning flow config topics...");
|
||||||
await ensureFlowConfig();
|
yield* ensureFlowConfig(gatewayUrl);
|
||||||
|
|
||||||
const finalFlows = [...(await listFlows())].sort();
|
const finalFlows = A.sort(Array.from(yield* listFlows(gatewayUrl)), Order.String);
|
||||||
console.log(`\nActive flows: ${finalFlows.join(", ")}`);
|
console.log(`\nActive flows: ${finalFlows.join(", ")}`);
|
||||||
console.log("\nFlow seeding complete.");
|
console.log("\nFlow seeding complete.");
|
||||||
}
|
|
||||||
|
|
||||||
main().catch((err) => {
|
|
||||||
console.error("Seed flows failed:", err);
|
|
||||||
process.exit(1);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
BunRuntime.runMain(main().pipe(Effect.provide(BunHttpClient.layer)));
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,15 @@
|
||||||
* Usage: pnpm tsx scripts/test-pipeline.ts
|
* Usage: pnpm tsx scripts/test-pipeline.ts
|
||||||
*/
|
*/
|
||||||
|
|
||||||
const GATEWAY_URL = process.env.GATEWAY_URL ?? "http://localhost:8088";
|
import { BunRuntime } from "@effect/platform-bun";
|
||||||
|
import * as BunHttpClient from "@effect/platform-bun/BunHttpClient";
|
||||||
|
import { Config, Effect, Option as O, Schema as S } from "effect";
|
||||||
|
import { HttpClient, HttpClientRequest } from "effect/unstable/http";
|
||||||
|
|
||||||
|
const DEFAULT_GATEWAY_URL = "http://localhost:8088";
|
||||||
|
const DEFAULT_LLM_MODEL = "qwen2.5:0.5b";
|
||||||
|
const DEFAULT_FALKORDB_URL = "redis://localhost:6380";
|
||||||
|
const DEFAULT_PIPELINE_WAIT = 20;
|
||||||
|
|
||||||
interface RpcSocket {
|
interface RpcSocket {
|
||||||
close: () => void;
|
close: () => void;
|
||||||
|
|
@ -22,24 +30,111 @@ interface RpcSocket {
|
||||||
) => Promise<ResponseType>;
|
) => Promise<ResponseType>;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Helpers ──────────────────────────────────────────────────────────
|
interface PipelineConfig {
|
||||||
|
readonly gatewayUrl: string;
|
||||||
async function post(path: string, body: unknown): Promise<unknown> {
|
readonly gatewaySecret: string | undefined;
|
||||||
const res = await fetch(`${GATEWAY_URL}${path}`, {
|
readonly llmModel: string;
|
||||||
method: "POST",
|
readonly pipelineWaitSeconds: number;
|
||||||
headers: { "Content-Type": "application/json" },
|
readonly falkorDbUrl: string;
|
||||||
body: JSON.stringify(body),
|
readonly skipPipeline: boolean;
|
||||||
});
|
readonly skipLlm: boolean;
|
||||||
const text = await res.text();
|
readonly skipLibrarian: boolean;
|
||||||
try {
|
readonly skipAgent: boolean;
|
||||||
return JSON.parse(text);
|
|
||||||
} catch {
|
|
||||||
return { status: res.status, body: text };
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class PipelineTestError extends S.TaggedErrorClass<PipelineTestError>()(
|
||||||
|
"PipelineTestError",
|
||||||
|
{
|
||||||
|
operation: S.String,
|
||||||
|
message: S.String,
|
||||||
|
},
|
||||||
|
) {}
|
||||||
|
|
||||||
|
const QdrantCollectionsResponse = S.Struct({
|
||||||
|
result: S.optionalKey(S.Struct({
|
||||||
|
collections: S.optionalKey(S.Array(S.Struct({ name: S.String }))),
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
|
||||||
|
const pipelineError = (operation: string, cause: unknown) =>
|
||||||
|
PipelineTestError.make({
|
||||||
|
operation,
|
||||||
|
message: String(cause),
|
||||||
|
});
|
||||||
|
|
||||||
|
const skipFlag = (name: string) =>
|
||||||
|
Config.string(name).pipe(
|
||||||
|
Config.withDefault("0"),
|
||||||
|
Config.map((value) => value === "1"),
|
||||||
|
);
|
||||||
|
|
||||||
|
const loadConfig = Effect.fn("test-pipeline.loadConfig")(function* () {
|
||||||
|
const gatewaySecret = yield* Config.string("GATEWAY_SECRET").pipe(Config.option);
|
||||||
|
return {
|
||||||
|
gatewayUrl: yield* Config.string("GATEWAY_URL").pipe(Config.withDefault(DEFAULT_GATEWAY_URL)),
|
||||||
|
gatewaySecret: O.getOrUndefined(gatewaySecret),
|
||||||
|
llmModel: yield* Config.string("LLM_MODEL").pipe(Config.withDefault(DEFAULT_LLM_MODEL)),
|
||||||
|
pipelineWaitSeconds: yield* Config.number("PIPELINE_WAIT").pipe(Config.withDefault(DEFAULT_PIPELINE_WAIT)),
|
||||||
|
falkorDbUrl: yield* Config.string("FALKORDB_URL").pipe(Config.withDefault(DEFAULT_FALKORDB_URL)),
|
||||||
|
skipPipeline: yield* skipFlag("SKIP_PIPELINE"),
|
||||||
|
skipLlm: yield* skipFlag("SKIP_LLM"),
|
||||||
|
skipLibrarian: yield* skipFlag("SKIP_LIBRARIAN"),
|
||||||
|
skipAgent: yield* skipFlag("SKIP_AGENT"),
|
||||||
|
} satisfies PipelineConfig;
|
||||||
|
});
|
||||||
|
|
||||||
|
// ─── Helpers ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const stringifyJson = (operation: string, value: unknown) =>
|
||||||
|
S.encodeUnknownEffect(S.UnknownFromJsonString)(value).pipe(
|
||||||
|
Effect.mapError((cause) => pipelineError(operation, cause)),
|
||||||
|
);
|
||||||
|
|
||||||
|
const decodeJsonText = (operation: string, value: string) =>
|
||||||
|
S.decodeUnknownEffect(S.UnknownFromJsonString)(value).pipe(
|
||||||
|
Effect.mapError((cause) => pipelineError(operation, cause)),
|
||||||
|
);
|
||||||
|
|
||||||
|
const post = Effect.fn("test-pipeline.post")(function* (
|
||||||
|
config: PipelineConfig,
|
||||||
|
path: string,
|
||||||
|
body: unknown,
|
||||||
|
) {
|
||||||
|
const bodyText = yield* stringifyJson("post.encode-request", body);
|
||||||
|
const request = HttpClientRequest.post(`${config.gatewayUrl}${path}`, { acceptJson: true }).pipe(
|
||||||
|
HttpClientRequest.bodyText(bodyText, "application/json"),
|
||||||
|
);
|
||||||
|
const response = yield* HttpClient.execute(request).pipe(
|
||||||
|
Effect.mapError((cause) => pipelineError("post.http", cause)),
|
||||||
|
);
|
||||||
|
const text = yield* response.text.pipe(
|
||||||
|
Effect.mapError((cause) => pipelineError("post.read-response", cause)),
|
||||||
|
);
|
||||||
|
return yield* decodeJsonText("post.decode-response", text).pipe(
|
||||||
|
Effect.catch(() => Effect.succeed({ status: response.status, body: text })),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
const getJson = Effect.fn("test-pipeline.getJson")(function* (url: string) {
|
||||||
|
const response = yield* HttpClient.get(url, { acceptJson: true }).pipe(
|
||||||
|
Effect.mapError((cause) => pipelineError("get.http", cause)),
|
||||||
|
);
|
||||||
|
const text = yield* response.text.pipe(
|
||||||
|
Effect.mapError((cause) => pipelineError("get.read-response", cause)),
|
||||||
|
);
|
||||||
|
return yield* decodeJsonText("get.decode-response", text);
|
||||||
|
});
|
||||||
|
|
||||||
|
const gatewayReachable = Effect.fn("test-pipeline.gatewayReachable")(function* (config: PipelineConfig) {
|
||||||
|
return yield* HttpClient.get(`${config.gatewayUrl}/api/v1/metrics`).pipe(
|
||||||
|
Effect.map((response) => response.status >= 200 && response.status < 300),
|
||||||
|
Effect.catch(() => Effect.succeed(false)),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
function log(label: string, data: unknown): void {
|
function log(label: string, data: unknown): void {
|
||||||
console.log(`\n[${label}]`, JSON.stringify(data, null, 2));
|
console.log(`\n[${label}]`);
|
||||||
|
console.dir(data, { depth: null });
|
||||||
}
|
}
|
||||||
|
|
||||||
function pass(test: string): void {
|
function pass(test: string): void {
|
||||||
|
|
@ -50,11 +145,18 @@ function fail(test: string, err: unknown): void {
|
||||||
console.error(` ✗ ${test}:`, err);
|
console.error(` ✗ ${test}:`, err);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const catchTest = <R, E>(name: string, effect: Effect.Effect<boolean, E, R>) =>
|
||||||
|
effect.pipe(
|
||||||
|
Effect.catch((err) => {
|
||||||
|
fail(name, err);
|
||||||
|
return Effect.succeed(false);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
// ─── Tests ────────────────────────────────────────────────────────────
|
// ─── Tests ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async function testConfigList(): Promise<boolean> {
|
const testConfigList = (config: PipelineConfig) => catchTest("Config list", Effect.gen(function* () {
|
||||||
try {
|
const res = yield* post(config, "/api/v1/config", { operation: "list", keys: [] });
|
||||||
const res = await post("/api/v1/config", { operation: "list", keys: [] });
|
|
||||||
log("config/list", res);
|
log("config/list", res);
|
||||||
if (typeof res === "object" && res !== null && "version" in res) {
|
if (typeof res === "object" && res !== null && "version" in res) {
|
||||||
pass("Config list returns version");
|
pass("Config list returns version");
|
||||||
|
|
@ -62,15 +164,10 @@ async function testConfigList(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Config list", "unexpected response");
|
fail("Config list", "unexpected response");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Config list", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testConfigPut(): Promise<boolean> {
|
const testConfigPut = (config: PipelineConfig) => catchTest("Config put", Effect.gen(function* () {
|
||||||
try {
|
const res = yield* post(config, "/api/v1/config", {
|
||||||
const res = await post("/api/v1/config", {
|
|
||||||
operation: "put",
|
operation: "put",
|
||||||
keys: ["test"],
|
keys: ["test"],
|
||||||
values: { greeting: "hello from trustgraph-ts!" },
|
values: { greeting: "hello from trustgraph-ts!" },
|
||||||
|
|
@ -82,15 +179,10 @@ async function testConfigPut(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Config put", "unexpected response");
|
fail("Config put", "unexpected response");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Config put", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testConfigGet(): Promise<boolean> {
|
const testConfigGet = (config: PipelineConfig) => catchTest("Config get", Effect.gen(function* () {
|
||||||
try {
|
const res = yield* post(config, "/api/v1/config", {
|
||||||
const res = await post("/api/v1/config", {
|
|
||||||
operation: "get",
|
operation: "get",
|
||||||
keys: ["test"],
|
keys: ["test"],
|
||||||
});
|
});
|
||||||
|
|
@ -103,22 +195,17 @@ async function testConfigGet(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Config get", "value mismatch");
|
fail("Config get", "value mismatch");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Config get", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testConfigDelete(): Promise<boolean> {
|
const testConfigDelete = (config: PipelineConfig) => catchTest("Config delete", Effect.gen(function* () {
|
||||||
try {
|
const res = yield* post(config, "/api/v1/config", {
|
||||||
const res = await post("/api/v1/config", {
|
|
||||||
operation: "delete",
|
operation: "delete",
|
||||||
keys: ["test"],
|
keys: ["test"],
|
||||||
});
|
});
|
||||||
log("config/delete", res);
|
log("config/delete", res);
|
||||||
|
|
||||||
// Verify it's gone
|
// Verify it's gone
|
||||||
const check = await post("/api/v1/config", {
|
const check = yield* post(config, "/api/v1/config", {
|
||||||
operation: "get",
|
operation: "get",
|
||||||
keys: ["test"],
|
keys: ["test"],
|
||||||
}) as Record<string, unknown>;
|
}) as Record<string, unknown>;
|
||||||
|
|
@ -130,16 +217,11 @@ async function testConfigDelete(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Config delete", "value still present");
|
fail("Config delete", "value still present");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Config delete", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testPushFlowConfig(): Promise<boolean> {
|
const testPushFlowConfig = (config: PipelineConfig) => catchTest("Flow config push", Effect.gen(function* () {
|
||||||
try {
|
|
||||||
// Push a full flow definition with all service topic mappings
|
// Push a full flow definition with all service topic mappings
|
||||||
const res = await post("/api/v1/config", {
|
const res = yield* post(config, "/api/v1/config", {
|
||||||
operation: "put",
|
operation: "put",
|
||||||
keys: ["flows"],
|
keys: ["flows"],
|
||||||
values: {
|
values: {
|
||||||
|
|
@ -193,21 +275,14 @@ async function testPushFlowConfig(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Flow config push", "unexpected response");
|
fail("Flow config push", "unexpected response");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Flow config push", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testTextCompletion(): Promise<boolean> {
|
const testTextCompletion = (config: PipelineConfig) => catchTest("Text completion", Effect.gen(function* () {
|
||||||
try {
|
|
||||||
console.log("\n Sending text-completion request (may take a few seconds)...");
|
console.log("\n Sending text-completion request (may take a few seconds)...");
|
||||||
// Use model from env or default to qwen2.5:0.5b (Ollama-compatible)
|
const res = yield* post(config, "/api/v1/flow/default/service/text-completion", {
|
||||||
const model = process.env.LLM_MODEL ?? "qwen2.5:0.5b";
|
|
||||||
const res = await post("/api/v1/flow/default/service/text-completion", {
|
|
||||||
system: "You are a helpful assistant. Reply in one sentence.",
|
system: "You are a helpful assistant. Reply in one sentence.",
|
||||||
prompt: "What is 2+2?",
|
prompt: "What is 2+2?",
|
||||||
model,
|
model: config.llmModel,
|
||||||
});
|
});
|
||||||
log("text-completion", res);
|
log("text-completion", res);
|
||||||
const r = res as Record<string, unknown>;
|
const r = res as Record<string, unknown>;
|
||||||
|
|
@ -221,55 +296,45 @@ async function testTextCompletion(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Text completion", "unexpected response");
|
fail("Text completion", "unexpected response");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Text completion", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testWebSocket(): Promise<boolean> {
|
const testWebSocket = (config: PipelineConfig) => catchTest("Effect RPC WebSocket", Effect.gen(function* () {
|
||||||
let socket: RpcSocket | undefined;
|
let socket: RpcSocket | undefined;
|
||||||
try {
|
return yield* Effect.gen(function* () {
|
||||||
const { createTrustGraphSocket } = await import(
|
const { createTrustGraphSocket } = yield* Effect.tryPromise({
|
||||||
"../packages/client/src/socket/trustgraph-socket.js"
|
try: () => import("../packages/client/src/socket/trustgraph-socket.js"),
|
||||||
);
|
catch: (cause) => pipelineError("websocket.import", cause),
|
||||||
|
});
|
||||||
|
|
||||||
const gatewayWsUrl = GATEWAY_URL.replace(/^http/, "ws").replace(/\/$/, "");
|
const gatewayWsUrl = config.gatewayUrl.replace(/^http/, "ws").replace(/\/$/, "");
|
||||||
socket = createTrustGraphSocket(
|
socket = createTrustGraphSocket(
|
||||||
"pipeline",
|
"pipeline",
|
||||||
process.env.GATEWAY_SECRET,
|
config.gatewaySecret,
|
||||||
`${gatewayWsUrl}/api/v1/rpc`,
|
`${gatewayWsUrl}/api/v1/rpc`,
|
||||||
);
|
);
|
||||||
const response = await Promise.race([
|
const response = yield* Effect.tryPromise({
|
||||||
socket.makeRequest<Record<string, unknown>, Record<string, unknown>>(
|
try: () =>
|
||||||
|
socket?.makeRequest<Record<string, unknown>, Record<string, unknown>>(
|
||||||
"config",
|
"config",
|
||||||
{ operation: "list", keys: [] },
|
{ operation: "list", keys: [] },
|
||||||
5000,
|
5000,
|
||||||
),
|
) ?? Promise.resolve({}),
|
||||||
new Promise<never>((_, reject) =>
|
catch: (cause) => pipelineError("websocket.request", cause),
|
||||||
setTimeout(() => reject(new Error("connection timeout")), 5000)
|
}).pipe(Effect.timeout("5 seconds"));
|
||||||
),
|
|
||||||
]);
|
|
||||||
|
|
||||||
log("websocket/rpc-response", response);
|
log("websocket/rpc-response", response);
|
||||||
pass("Effect RPC WebSocket round-trip works");
|
pass("Effect RPC WebSocket round-trip works");
|
||||||
return true;
|
return true;
|
||||||
} catch (err) {
|
}).pipe(Effect.ensuring(Effect.sync(() => socket?.close())));
|
||||||
fail("Effect RPC WebSocket", err);
|
}));
|
||||||
return false;
|
|
||||||
} finally {
|
|
||||||
socket?.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Librarian Tests ──────────────────────────────────────────────────
|
// ─── Librarian Tests ──────────────────────────────────────────────────
|
||||||
|
|
||||||
let testDocId = "";
|
let testDocId = "";
|
||||||
|
|
||||||
async function testLibrarianAdd(): Promise<boolean> {
|
const testLibrarianAdd = (config: PipelineConfig) => catchTest("Librarian add-document", Effect.gen(function* () {
|
||||||
try {
|
|
||||||
const content = Buffer.from("Hello from TrustGraph TypeScript!").toString("base64");
|
const content = Buffer.from("Hello from TrustGraph TypeScript!").toString("base64");
|
||||||
const res = await post("/api/v1/librarian", {
|
const res = yield* post(config, "/api/v1/librarian", {
|
||||||
operation: "add-document",
|
operation: "add-document",
|
||||||
user: "test-user",
|
user: "test-user",
|
||||||
collection: "test-collection",
|
collection: "test-collection",
|
||||||
|
|
@ -299,15 +364,10 @@ async function testLibrarianAdd(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Librarian add-document", "no documentMetadata.id in response");
|
fail("Librarian add-document", "no documentMetadata.id in response");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Librarian add-document", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testLibrarianList(): Promise<boolean> {
|
const testLibrarianList = (config: PipelineConfig) => catchTest("Librarian list-documents", Effect.gen(function* () {
|
||||||
try {
|
const res = yield* post(config, "/api/v1/librarian", {
|
||||||
const res = await post("/api/v1/librarian", {
|
|
||||||
operation: "list-documents",
|
operation: "list-documents",
|
||||||
user: "test-user",
|
user: "test-user",
|
||||||
});
|
});
|
||||||
|
|
@ -320,19 +380,14 @@ async function testLibrarianList(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Librarian list-documents", "empty or missing documents array");
|
fail("Librarian list-documents", "empty or missing documents array");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Librarian list-documents", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testLibrarianGetContent(): Promise<boolean> {
|
const testLibrarianGetContent = (config: PipelineConfig) => catchTest("Librarian get-content", Effect.gen(function* () {
|
||||||
if (!testDocId) {
|
if (!testDocId) {
|
||||||
fail("Librarian get-content", "no document ID from add test");
|
fail("Librarian get-content", "no document ID from add test");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
try {
|
const res = yield* post(config, "/api/v1/librarian", {
|
||||||
const res = await post("/api/v1/librarian", {
|
|
||||||
operation: "get-document-content",
|
operation: "get-document-content",
|
||||||
documentId: testDocId,
|
documentId: testDocId,
|
||||||
user: "test-user",
|
user: "test-user",
|
||||||
|
|
@ -350,19 +405,14 @@ async function testLibrarianGetContent(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Librarian get-content", "no content in response");
|
fail("Librarian get-content", "no content in response");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Librarian get-content", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testLibrarianDelete(): Promise<boolean> {
|
const testLibrarianDelete = (config: PipelineConfig) => catchTest("Librarian delete", Effect.gen(function* () {
|
||||||
if (!testDocId) {
|
if (!testDocId) {
|
||||||
fail("Librarian delete", "no document ID from add test");
|
fail("Librarian delete", "no document ID from add test");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
try {
|
const res = yield* post(config, "/api/v1/librarian", {
|
||||||
const res = await post("/api/v1/librarian", {
|
|
||||||
operation: "remove-document",
|
operation: "remove-document",
|
||||||
documentId: testDocId,
|
documentId: testDocId,
|
||||||
user: "test-user",
|
user: "test-user",
|
||||||
|
|
@ -370,7 +420,7 @@ async function testLibrarianDelete(): Promise<boolean> {
|
||||||
log("librarian/delete", res);
|
log("librarian/delete", res);
|
||||||
|
|
||||||
// Verify it's gone
|
// Verify it's gone
|
||||||
const listRes = await post("/api/v1/librarian", {
|
const listRes = yield* post(config, "/api/v1/librarian", {
|
||||||
operation: "list-documents",
|
operation: "list-documents",
|
||||||
user: "test-user",
|
user: "test-user",
|
||||||
}) as Record<string, unknown>;
|
}) as Record<string, unknown>;
|
||||||
|
|
@ -381,19 +431,14 @@ async function testLibrarianDelete(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Librarian remove-document", "document still present after delete");
|
fail("Librarian remove-document", "document still present after delete");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Librarian delete", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Document Load Test ──────────────────────────────────────────────
|
// ─── Document Load Test ──────────────────────────────────────────────
|
||||||
|
|
||||||
async function testDocumentLoad(): Promise<boolean> {
|
const testDocumentLoad = (config: PipelineConfig) => catchTest("Document load", Effect.gen(function* () {
|
||||||
try {
|
|
||||||
// First upload a test document via librarian
|
// First upload a test document via librarian
|
||||||
const content = Buffer.from("Test document for pipeline processing.").toString("base64");
|
const content = Buffer.from("Test document for pipeline processing.").toString("base64");
|
||||||
const addRes = await post("/api/v1/librarian", {
|
const addRes = yield* post(config, "/api/v1/librarian", {
|
||||||
operation: "add-document",
|
operation: "add-document",
|
||||||
user: "test-user",
|
user: "test-user",
|
||||||
collection: "test-collection",
|
collection: "test-collection",
|
||||||
|
|
@ -418,23 +463,18 @@ async function testDocumentLoad(): Promise<boolean> {
|
||||||
const docId = meta.id as string;
|
const docId = meta.id as string;
|
||||||
|
|
||||||
// Trigger document processing via the load endpoint
|
// Trigger document processing via the load endpoint
|
||||||
const res = await fetch(`${GATEWAY_URL}/api/v1/flow/default/load`, {
|
const data = yield* post(config, "/api/v1/flow/default/load", {
|
||||||
method: "POST",
|
documentId: docId,
|
||||||
headers: { "Content-Type": "application/json" },
|
user: "test-user",
|
||||||
body: JSON.stringify({
|
collection: "test-collection",
|
||||||
documentId: docId,
|
|
||||||
user: "test-user",
|
|
||||||
collection: "test-collection",
|
|
||||||
}),
|
|
||||||
});
|
});
|
||||||
const data = await res.json() as Record<string, unknown>;
|
|
||||||
log("document-load", data);
|
log("document-load", data);
|
||||||
|
|
||||||
if (data.status === "processing") {
|
if (data.status === "processing") {
|
||||||
pass(`Document load triggered for ${docId.slice(0, 8)}...`);
|
pass(`Document load triggered for ${docId.slice(0, 8)}...`);
|
||||||
|
|
||||||
// Clean up the test document
|
// Clean up the test document
|
||||||
await post("/api/v1/librarian", {
|
yield* post(config, "/api/v1/librarian", {
|
||||||
operation: "remove-document",
|
operation: "remove-document",
|
||||||
documentId: docId,
|
documentId: docId,
|
||||||
user: "test-user",
|
user: "test-user",
|
||||||
|
|
@ -444,21 +484,25 @@ async function testDocumentLoad(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Document load", "unexpected response");
|
fail("Document load", "unexpected response");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Document load", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Full Pipeline Test (real PDF) ───────────────────────────────────
|
// ─── Full Pipeline Test (real PDF) ───────────────────────────────────
|
||||||
|
|
||||||
async function testFullPipeline(): Promise<boolean> {
|
const testFullPipeline = (config: PipelineConfig) => catchTest("Full pipeline", Effect.gen(function* () {
|
||||||
try {
|
|
||||||
// 1. Generate a test PDF in memory using pdf-lib
|
// 1. Generate a test PDF in memory using pdf-lib
|
||||||
const { PDFDocument, StandardFonts } = await import("pdf-lib");
|
const { PDFDocument, StandardFonts } = yield* Effect.tryPromise({
|
||||||
|
try: () => import("pdf-lib"),
|
||||||
|
catch: (cause) => pipelineError("full-pipeline.import-pdf-lib", cause),
|
||||||
|
});
|
||||||
|
|
||||||
const pdfDoc = await PDFDocument.create();
|
const pdfDoc = yield* Effect.tryPromise({
|
||||||
const font = await pdfDoc.embedFont(StandardFonts.Helvetica);
|
try: () => PDFDocument.create(),
|
||||||
|
catch: (cause) => pipelineError("full-pipeline.create-pdf", cause),
|
||||||
|
});
|
||||||
|
const font = yield* Effect.tryPromise({
|
||||||
|
try: () => pdfDoc.embedFont(StandardFonts.Helvetica),
|
||||||
|
catch: (cause) => pipelineError("full-pipeline.embed-font", cause),
|
||||||
|
});
|
||||||
|
|
||||||
const texts = [
|
const texts = [
|
||||||
"Alice Johnson is a senior engineer at Acme Corporation. Acme develops CloudSync, a cloud storage platform. CloudSync uses Amazon Web Services for hosting.",
|
"Alice Johnson is a senior engineer at Acme Corporation. Acme develops CloudSync, a cloud storage platform. CloudSync uses Amazon Web Services for hosting.",
|
||||||
|
|
@ -470,13 +514,16 @@ async function testFullPipeline(): Promise<boolean> {
|
||||||
page.drawText(text, { x: 50, y: 700, size: 11, font, maxWidth: 500 });
|
page.drawText(text, { x: 50, y: 700, size: 11, font, maxWidth: 500 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const pdfBytes = await pdfDoc.save();
|
const pdfBytes = yield* Effect.tryPromise({
|
||||||
|
try: () => pdfDoc.save(),
|
||||||
|
catch: (cause) => pipelineError("full-pipeline.save-pdf", cause),
|
||||||
|
});
|
||||||
const content = Buffer.from(pdfBytes).toString("base64");
|
const content = Buffer.from(pdfBytes).toString("base64");
|
||||||
|
|
||||||
console.log(` Generated test PDF: ${pdfBytes.length} bytes, 2 pages`);
|
console.log(` Generated test PDF: ${pdfBytes.length} bytes, 2 pages`);
|
||||||
|
|
||||||
// 2. Upload to librarian as application/pdf
|
// 2. Upload to librarian as application/pdf
|
||||||
const addRes = await post("/api/v1/librarian", {
|
const addRes = yield* post(config, "/api/v1/librarian", {
|
||||||
operation: "add-document",
|
operation: "add-document",
|
||||||
user: "test",
|
user: "test",
|
||||||
collection: "test",
|
collection: "test",
|
||||||
|
|
@ -502,61 +549,76 @@ async function testFullPipeline(): Promise<boolean> {
|
||||||
console.log(` Uploaded PDF as document ${docId.slice(0, 8)}...`);
|
console.log(` Uploaded PDF as document ${docId.slice(0, 8)}...`);
|
||||||
|
|
||||||
// 3. Trigger pipeline processing
|
// 3. Trigger pipeline processing
|
||||||
const loadRes = await fetch(`${GATEWAY_URL}/api/v1/flow/default/load`, {
|
const loadData = yield* post(config, "/api/v1/flow/default/load", {
|
||||||
method: "POST",
|
documentId: docId,
|
||||||
headers: { "Content-Type": "application/json" },
|
user: "test",
|
||||||
body: JSON.stringify({ documentId: docId, user: "test", collection: "test" }),
|
collection: "test",
|
||||||
});
|
});
|
||||||
const loadData = await loadRes.json() as Record<string, unknown>;
|
|
||||||
|
|
||||||
if (loadData.status !== "processing") {
|
const loadRecord = loadData as Record<string, unknown>;
|
||||||
fail("Full pipeline", `load returned: ${JSON.stringify(loadData)}`);
|
if (loadRecord.status !== "processing") {
|
||||||
|
fail("Full pipeline", `load returned: ${String(loadData)}`);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
console.log(" Pipeline triggered, waiting for processing...");
|
console.log(" Pipeline triggered, waiting for processing...");
|
||||||
|
|
||||||
// 4. Wait for pipeline to complete (PDF decode + chunking + extraction + storage)
|
// 4. Wait for pipeline to complete (PDF decode + chunking + extraction + storage)
|
||||||
// This involves multiple LLM calls so give it time
|
// This involves multiple LLM calls so give it time
|
||||||
const waitSecs = Number.parseInt(process.env.PIPELINE_WAIT ?? "20", 10);
|
for (let i = config.pipelineWaitSeconds; i > 0; i--) {
|
||||||
for (let i = waitSecs; i > 0; i--) {
|
|
||||||
process.stdout.write(`\r Waiting... ${i}s remaining `);
|
process.stdout.write(`\r Waiting... ${i}s remaining `);
|
||||||
await new Promise((r) => setTimeout(r, 1000));
|
yield* Effect.sleep("1 second");
|
||||||
}
|
}
|
||||||
console.log("\r Processing wait complete. ");
|
console.log("\r Processing wait complete. ");
|
||||||
|
|
||||||
// 5. Verify triples in FalkorDB
|
// 5. Verify triples in FalkorDB
|
||||||
let triplesFound = false;
|
let triplesFound = false;
|
||||||
try {
|
const falkorCount = yield* Effect.tryPromise({
|
||||||
const { createClient } = await import("falkordb");
|
try: async () => {
|
||||||
const client = createClient({
|
const { createClient } = await import("falkordb");
|
||||||
url: process.env.FALKORDB_URL ?? "redis://localhost:6380",
|
const client = createClient({
|
||||||
});
|
url: config.falkorDbUrl,
|
||||||
await client.connect();
|
});
|
||||||
const graph = client.graph("falkordb");
|
await client.connect();
|
||||||
const result = await graph.query("MATCH (n:Node) RETURN count(n) as cnt");
|
const graph = client.graph("falkordb");
|
||||||
const count = result.data?.[0]?.[0] ?? 0;
|
const result = await graph.query("MATCH (n:Node) RETURN count(n) as cnt");
|
||||||
await client.disconnect();
|
const count = result.data?.[0]?.[0] ?? 0;
|
||||||
|
await client.disconnect();
|
||||||
|
return count;
|
||||||
|
},
|
||||||
|
catch: (cause) => pipelineError("full-pipeline.falkordb", cause),
|
||||||
|
}).pipe(
|
||||||
|
Effect.catch((err) => {
|
||||||
|
const errStr = String(err);
|
||||||
|
if (errStr.includes("Cannot find package") || errStr.includes("MODULE_NOT_FOUND")) {
|
||||||
|
console.log(" FalkorDB check skipped: falkordb package not available at workspace root");
|
||||||
|
} else {
|
||||||
|
console.log(` FalkorDB check failed: ${err}`);
|
||||||
|
}
|
||||||
|
return Effect.succeed(undefined);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
if (typeof count === "number" && count > 0) {
|
if (typeof falkorCount === "number" && falkorCount > 0) {
|
||||||
console.log(` FalkorDB: ${count} nodes found`);
|
console.log(` FalkorDB: ${falkorCount} nodes found`);
|
||||||
triplesFound = true;
|
triplesFound = true;
|
||||||
} else {
|
} else {
|
||||||
console.log(` FalkorDB: no nodes found (count=${count})`);
|
console.log(` FalkorDB: no nodes found (count=${falkorCount})`);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
|
||||||
const errStr = String(err);
|
|
||||||
if (errStr.includes("Cannot find package") || errStr.includes("MODULE_NOT_FOUND")) {
|
|
||||||
console.log(" FalkorDB check skipped: falkordb package not available at workspace root");
|
|
||||||
} else {
|
|
||||||
console.log(` FalkorDB check failed: ${err}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 6. Verify embeddings in Qdrant
|
// 6. Verify embeddings in Qdrant
|
||||||
let embeddingsFound = false;
|
let embeddingsFound = false;
|
||||||
try {
|
const qdrantData = yield* getJson("http://localhost:6333/collections").pipe(
|
||||||
const qdrantRes = await fetch("http://localhost:6333/collections");
|
Effect.flatMap((value) =>
|
||||||
const qdrantData = await qdrantRes.json() as { result?: { collections?: Array<{ name: string }> } };
|
S.decodeUnknownEffect(QdrantCollectionsResponse)(value).pipe(
|
||||||
|
Effect.mapError((cause) => pipelineError("full-pipeline.qdrant.decode", cause)),
|
||||||
|
)
|
||||||
|
),
|
||||||
|
Effect.catch((err) => {
|
||||||
|
console.log(` Qdrant check failed: ${err}`);
|
||||||
|
return Effect.succeed(undefined);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
if (qdrantData !== undefined) {
|
||||||
const collections = qdrantData.result?.collections ?? [];
|
const collections = qdrantData.result?.collections ?? [];
|
||||||
const testCollections = collections.filter((c) => c.name.startsWith("t_test_test_"));
|
const testCollections = collections.filter((c) => c.name.startsWith("t_test_test_"));
|
||||||
|
|
||||||
|
|
@ -566,8 +628,6 @@ async function testFullPipeline(): Promise<boolean> {
|
||||||
} else {
|
} else {
|
||||||
console.log(` Qdrant: no test collections found (total: ${collections.length} collections)`);
|
console.log(` Qdrant: no test collections found (total: ${collections.length} collections)`);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
|
||||||
console.log(` Qdrant check failed: ${err}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 7. Report results
|
// 7. Report results
|
||||||
|
|
@ -584,21 +644,15 @@ async function testFullPipeline(): Promise<boolean> {
|
||||||
// Pipeline triggered but stores not populated yet — partial success
|
// Pipeline triggered but stores not populated yet — partial success
|
||||||
pass("Full pipeline: triggered successfully (stores may need more time)");
|
pass("Full pipeline: triggered successfully (stores may need more time)");
|
||||||
return true;
|
return true;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Full pipeline", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Agent Test ───────────────────────────────────────────────────────
|
// ─── Agent Test ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
async function testAgentQuery(): Promise<boolean> {
|
const testAgentQuery = (config: PipelineConfig) => catchTest("Agent", Effect.gen(function* () {
|
||||||
try {
|
|
||||||
console.log("\n Sending agent request (may take a few seconds)...");
|
console.log("\n Sending agent request (may take a few seconds)...");
|
||||||
const model = process.env.LLM_MODEL ?? "qwen2.5:0.5b";
|
const res = yield* post(config, "/api/v1/flow/default/service/agent", {
|
||||||
const res = await post("/api/v1/flow/default/service/agent", {
|
|
||||||
question: "What is the capital of France?",
|
question: "What is the capital of France?",
|
||||||
model,
|
model: config.llmModel,
|
||||||
});
|
});
|
||||||
log("agent", res);
|
log("agent", res);
|
||||||
const r = res as Record<string, unknown>;
|
const r = res as Record<string, unknown>;
|
||||||
|
|
@ -615,91 +669,94 @@ async function testAgentQuery(): Promise<boolean> {
|
||||||
}
|
}
|
||||||
fail("Agent", "unexpected response format");
|
fail("Agent", "unexpected response format");
|
||||||
return false;
|
return false;
|
||||||
} catch (err) {
|
}));
|
||||||
fail("Agent", err);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ─── Main ─────────────────────────────────────────────────────────────
|
// ─── Main ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async function main(): Promise<void> {
|
const main = Effect.fn("test-pipeline.main")(function* () {
|
||||||
|
const config = yield* loadConfig();
|
||||||
|
|
||||||
console.log("╔══════════════════════════════════════════════════╗");
|
console.log("╔══════════════════════════════════════════════════╗");
|
||||||
console.log("║ TrustGraph TypeScript — Integration Test ║");
|
console.log("║ TrustGraph TypeScript — Integration Test ║");
|
||||||
console.log("╚══════════════════════════════════════════════════╝");
|
console.log("╚══════════════════════════════════════════════════╝");
|
||||||
console.log(`\nGateway: ${GATEWAY_URL}`);
|
console.log(`\nGateway: ${config.gatewayUrl}`);
|
||||||
|
|
||||||
// Check gateway is reachable
|
// Check gateway is reachable
|
||||||
try {
|
const isReachable = yield* gatewayReachable(config);
|
||||||
const res = await fetch(`${GATEWAY_URL}/api/v1/metrics`);
|
if (isReachable) {
|
||||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
||||||
pass("Gateway reachable");
|
pass("Gateway reachable");
|
||||||
} catch (err) {
|
} else {
|
||||||
fail("Gateway reachable", err);
|
fail("Gateway reachable", "metrics endpoint unavailable");
|
||||||
console.error("\n⚠ Gateway not running. Start it first:");
|
console.error("\n⚠ Gateway not running. Start it first:");
|
||||||
console.error(" pnpm tsx scripts/run-gateway.ts");
|
console.error(" pnpm tsx scripts/run-gateway.ts");
|
||||||
process.exit(1);
|
return yield* PipelineTestError.make({
|
||||||
|
operation: "gateway-reachable",
|
||||||
|
message: "gateway metrics endpoint unavailable",
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let passed = 0;
|
let passed = 0;
|
||||||
let failed = 0;
|
let failed = 0;
|
||||||
const run = async (name: string, fn: () => Promise<boolean>) => {
|
const run = Effect.fn("test-pipeline.run")(function* (
|
||||||
|
name: string,
|
||||||
|
test: Effect.Effect<boolean, never, HttpClient.HttpClient>,
|
||||||
|
) {
|
||||||
console.log(`\n── ${name} ──`);
|
console.log(`\n── ${name} ──`);
|
||||||
if (await fn()) passed++;
|
if (yield* test) passed++;
|
||||||
else failed++;
|
else failed++;
|
||||||
};
|
});
|
||||||
|
|
||||||
// Config CRUD tests
|
// Config CRUD tests
|
||||||
await run("Config List", testConfigList);
|
yield* run("Config List", testConfigList(config));
|
||||||
await run("Config Put", testConfigPut);
|
yield* run("Config Put", testConfigPut(config));
|
||||||
await run("Config Get", testConfigGet);
|
yield* run("Config Get", testConfigGet(config));
|
||||||
await run("Config Delete", testConfigDelete);
|
yield* run("Config Delete", testConfigDelete(config));
|
||||||
|
|
||||||
// WebSocket test
|
// WebSocket test
|
||||||
await run("WebSocket Round-Trip", testWebSocket);
|
yield* run("WebSocket Round-Trip", testWebSocket(config));
|
||||||
|
|
||||||
// Flow config push
|
// Flow config push
|
||||||
await run("Push Flow Config", testPushFlowConfig);
|
yield* run("Push Flow Config", testPushFlowConfig(config));
|
||||||
|
|
||||||
// Document pipeline load test (requires librarian + gateway)
|
// Document pipeline load test (requires librarian + gateway)
|
||||||
if (process.env.SKIP_PIPELINE !== "1" && process.env.SKIP_LIBRARIAN !== "1") {
|
if (!config.skipPipeline && !config.skipLibrarian) {
|
||||||
console.log("\n (Testing document load — set SKIP_PIPELINE=1 to skip)");
|
console.log("\n (Testing document load — set SKIP_PIPELINE=1 to skip)");
|
||||||
await run("Document Load", testDocumentLoad);
|
yield* run("Document Load", testDocumentLoad(config));
|
||||||
} else {
|
} else {
|
||||||
console.log("\n (Skipping document pipeline load test)");
|
console.log("\n (Skipping document pipeline load test)");
|
||||||
}
|
}
|
||||||
|
|
||||||
// LLM test (only if a running LLM service is available)
|
// LLM test (only if a running LLM service is available)
|
||||||
if (process.env.SKIP_LLM !== "1") {
|
if (!config.skipLlm) {
|
||||||
console.log("\n (Testing text-completion — set SKIP_LLM=1 to skip)");
|
console.log("\n (Testing text-completion — set SKIP_LLM=1 to skip)");
|
||||||
await run("Text Completion", testTextCompletion);
|
yield* run("Text Completion", testTextCompletion(config));
|
||||||
} else {
|
} else {
|
||||||
console.log("\n (SKIP_LLM=1 — skipping LLM test)");
|
console.log("\n (SKIP_LLM=1 — skipping LLM test)");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Librarian tests (only if librarian service is running)
|
// Librarian tests (only if librarian service is running)
|
||||||
if (process.env.SKIP_LIBRARIAN !== "1") {
|
if (!config.skipLibrarian) {
|
||||||
console.log("\n (Testing librarian — set SKIP_LIBRARIAN=1 to skip)");
|
console.log("\n (Testing librarian — set SKIP_LIBRARIAN=1 to skip)");
|
||||||
await run("Librarian Add", testLibrarianAdd);
|
yield* run("Librarian Add", testLibrarianAdd(config));
|
||||||
await run("Librarian List", testLibrarianList);
|
yield* run("Librarian List", testLibrarianList(config));
|
||||||
await run("Librarian Get Content", testLibrarianGetContent);
|
yield* run("Librarian Get Content", testLibrarianGetContent(config));
|
||||||
await run("Librarian Delete", testLibrarianDelete);
|
yield* run("Librarian Delete", testLibrarianDelete(config));
|
||||||
} else {
|
} else {
|
||||||
console.log("\n (SKIP_LIBRARIAN=1 — skipping librarian tests)");
|
console.log("\n (SKIP_LIBRARIAN=1 — skipping librarian tests)");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Full pipeline test (real PDF → decode → chunk → extract → store)
|
// Full pipeline test (real PDF → decode → chunk → extract → store)
|
||||||
if (process.env.SKIP_PIPELINE !== "1" && process.env.SKIP_LLM !== "1") {
|
if (!config.skipPipeline && !config.skipLlm) {
|
||||||
console.log("\n (Testing full pipeline with real PDF — set SKIP_PIPELINE=1 to skip)");
|
console.log("\n (Testing full pipeline with real PDF — set SKIP_PIPELINE=1 to skip)");
|
||||||
await run("Full Pipeline", testFullPipeline);
|
yield* run("Full Pipeline", testFullPipeline(config));
|
||||||
} else {
|
} else {
|
||||||
console.log("\n (Skipping full pipeline test)");
|
console.log("\n (Skipping full pipeline test)");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Agent test (only if agent + LLM services are running)
|
// Agent test (only if agent + LLM services are running)
|
||||||
if (process.env.SKIP_AGENT !== "1" && process.env.SKIP_LLM !== "1") {
|
if (!config.skipAgent && !config.skipLlm) {
|
||||||
console.log("\n (Testing agent — set SKIP_AGENT=1 to skip)");
|
console.log("\n (Testing agent — set SKIP_AGENT=1 to skip)");
|
||||||
await run("Agent Query", testAgentQuery);
|
yield* run("Agent Query", testAgentQuery(config));
|
||||||
} else {
|
} else {
|
||||||
console.log("\n (Skipping agent test)");
|
console.log("\n (Skipping agent test)");
|
||||||
}
|
}
|
||||||
|
|
@ -708,7 +765,12 @@ async function main(): Promise<void> {
|
||||||
console.log(` Results: ${passed} passed, ${failed} failed`);
|
console.log(` Results: ${passed} passed, ${failed} failed`);
|
||||||
console.log("══════════════════════════════════════════════════\n");
|
console.log("══════════════════════════════════════════════════\n");
|
||||||
|
|
||||||
process.exit(failed > 0 ? 1 : 0);
|
if (failed > 0) {
|
||||||
}
|
return yield* PipelineTestError.make({
|
||||||
|
operation: "results",
|
||||||
|
message: `${failed} integration test(s) failed`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
main();
|
BunRuntime.runMain(main().pipe(Effect.provide(BunHttpClient.layer)));
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue