apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-01 09:29:38 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat(ts): add real quality gates — Biome lint + effect-law ratchet + class inventory

Browse source 
- biome.json (2.4.16, linter-only) wired as "lint" in all six packages
- scripts/check-effect-laws.ts: Effect-native law enforcement encoding the
  adapted beep-effect effect-first/schema-first laws (no native JSON/switch/
  sort/fetch/timers, no process.env, no throw new, no Effect.run* outside
  boundaries, no Schema-suffixed constants, no node:fs/path, AST-based
  pure-data interface detection per law 38/39)
- ratcheting baseline allowlist (95 entries / 290 findings) that must shrink
  to documented exemptions only; stale counts fail the gate
- root lint chains turbo lint + law check + native-class inventory
- fix all 163 initial Biome findings: import-type style, templates, two `any`s,
  ten non-null assertions (librarian getService gate, A.matchRight in atoms,
  ensureNode returning nodes, main.tsx mount guard)

Gates: lint, check:tsgo, build, test (force, 11 tasks) all green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
elpresidank 2026-06-11 06:40:01 -05:00
parent cf12defcd8
commit 0746d7ffd5
109 changed files with 951 additions and 611 deletions
Download patch file Download diff file Expand all files Collapse all files

3
ts/packages/base/package.json
View file

@ -17,7 +17,8 @@
"build": "bunx --bun tsc",
"dev": "tsc --watch",
"clean": "rm -rf dist",
"test": "bunx --bun vitest run"
"test": "bunx --bun vitest run",
"lint": "bunx --bun biome check src"
},
"dependencies": {
"effect": "4.0.0-beta.78",

5
ts/packages/base/src/__tests__/consumer.test.ts
View file

@ -1,13 +1,12 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
import { Effect } from "effect";
import { makeConsumer, type ConsumerOptions, type FlowContext } from "../messaging/consumer.js";
import type { FlowContext } from "../messaging/consumer.js";
import { makeConsumer, } from "../messaging/consumer.js";
import type {
PubSubBackend,
BackendConsumer,
Message,
BackendProducer,
CreateProducerOptions,
CreateConsumerOptions,
} from "../backend/types.js";
import { tooManyRequestsError } from "../errors.js";
import type { Flow } from "../processor/flow.js";

18
ts/packages/base/src/__tests__/embeddings-service.test.ts
View file

@ -1,6 +1,16 @@
import { describe, expect, it } from "@effect/vitest";
import { ConfigProvider, Effect, Fiber } from "effect";
import * as S from "effect/Schema";
import type {
BackendConsumer,
BackendProducer,
CreateConsumerOptions,
CreateProducerOptions,
EmbeddingsRequest,
EmbeddingsResponse,
Message,
PubSubBackend,
} from "../index.js";
import {
Embeddings,
EmbeddingsService,
@ -9,14 +19,6 @@ import {
embeddingsError,
runProcessorScoped,
topics,
type BackendConsumer,
type BackendProducer,
type CreateConsumerOptions,
type CreateProducerOptions,
type EmbeddingsRequest,
type EmbeddingsResponse,
type Message,
type PubSubBackend,
} from "../index.js";
class WaitForTimeout extends S.TaggedErrorClass<WaitForTimeout>()(

16
ts/packages/base/src/__tests__/flow-processor-runtime.test.ts
View file

@ -1,6 +1,15 @@
import { describe, expect, it } from "@effect/vitest";
import { ConfigProvider, Effect, Fiber } from "effect";
import * as S from "effect/Schema";
import type {
BackendConsumer,
BackendProducer,
CreateConsumerOptions,
CreateProducerOptions,
Message,
ProcessorConfig,
PubSubBackend,
} from "../index.js";
import {
FlowProcessor,
MessagingRuntimeLive,
@ -9,13 +18,6 @@ import {
runFlowProcessorDefinitionScoped,
runProcessorScoped,
topics,
type BackendConsumer,
type BackendProducer,
type CreateConsumerOptions,
type CreateProducerOptions,
type Message,
type ProcessorConfig,
type PubSubBackend,
} from "../index.js";
class WaitForTimeout extends S.TaggedErrorClass<WaitForTimeout>()(

16
ts/packages/base/src/__tests__/flow-spec-runtime.test.ts
View file

@ -2,6 +2,15 @@ import { describe, expect, it } from "@effect/vitest";
import { ConfigProvider, Duration, Effect, Fiber } from "effect";
import * as S from "effect/Schema";
import * as TestClock from "effect/testing/TestClock";
import type {
BackendConsumer,
BackendProducer,
CreateConsumerOptions,
CreateProducerOptions,
FlowContext,
Message,
PubSubBackend,
} from "../index.js";
import {
makeConsumerSpec,
Flow,
@ -10,13 +19,6 @@ import {
makeProducerSpec,
PubSub,
makeRequestResponseSpec,
type BackendConsumer,
type BackendProducer,
type CreateConsumerOptions,
type CreateProducerOptions,
type FlowContext,
type Message,
type PubSubBackend,
} from "../index.js";
function createMessage<T>(value: T, properties: Record<string, string> = {}): Message<T> {

16
ts/packages/base/src/__tests__/messaging-runtime.test.ts
View file

@ -1,6 +1,15 @@
import { describe, expect, it } from "@effect/vitest";
import { Duration, Effect, Fiber } from "effect";
import * as TestClock from "effect/testing/TestClock";
import type {
BackendConsumer,
BackendProducer,
CreateConsumerOptions,
CreateProducerOptions,
FlowContext,
Message,
PubSubBackend,
} from "../index.js";
import {
PubSub,
defaultMessagingRuntimeConfig,
@ -11,13 +20,6 @@ import {
runEffectProducerScoped,
runFlowScoped,
tooManyRequestsError,
type BackendConsumer,
type BackendProducer,
type CreateConsumerOptions,
type CreateProducerOptions,
type FlowContext,
type Message,
type PubSubBackend,
} from "../index.js";
import type { Flow } from "../processor/flow.js";
import { Flow as RuntimeFlow } from "../processor/flow.js";

12
ts/packages/base/src/__tests__/producer.test.ts
View file

@ -1,13 +1,15 @@
import { describe, expect, it } from "vitest";
import { Effect } from "effect";
import type {
BackendConsumer,
BackendProducer,
CreateConsumerOptions,
CreateProducerOptions,
PubSubBackend,
} from "../index.js";
import {
makeProducer,
pubSubError,
type BackendConsumer,
type BackendProducer,
type CreateConsumerOptions,
type CreateProducerOptions,
type PubSubBackend,
} from "../index.js";
class ProducerBackend implements PubSubBackend {

14
ts/packages/base/src/__tests__/request-response.test.ts
View file

@ -1,13 +1,15 @@
import { describe, expect, it } from "vitest";
import { Effect } from "effect";
import type {
BackendConsumer,
BackendProducer,
CreateConsumerOptions,
CreateProducerOptions,
Message,
PubSubBackend,
} from "../index.js";
import {
makeRequestResponse,
type BackendConsumer,
type BackendProducer,
type CreateConsumerOptions,
type CreateProducerOptions,
type Message,
type PubSubBackend,
} from "../index.js";
function createMessage<T>(value: T, properties: Record<string, string> = {}): Message<T> {

16
ts/packages/base/src/__tests__/runtime-services.test.ts
View file

@ -1,18 +1,20 @@
import { describe, expect, it } from "@effect/vitest";
import { Effect } from "effect";
import * as S from "effect/Schema";
import type {
BackendConsumer,
BackendProducer,
CreateConsumerOptions,
CreateProducerOptions,
Message,
ProcessorConfig,
PubSubBackend,
} from "../index.js";
import {
PubSub,
makeAsyncProcessor,
pubSubError,
runProcessorScoped,
type BackendConsumer,
type BackendProducer,
type CreateConsumerOptions,
type CreateProducerOptions,
type Message,
type ProcessorConfig,
type PubSubBackend,
} from "../index.js";
class RuntimeServicesTestError extends S.TaggedErrorClass<RuntimeServicesTestError>()(

17
ts/packages/base/src/backend/nats.ts
View file

@ -8,16 +8,18 @@
* Python reference: trustgraph-base/trustgraph/base/pulsar_backend.py
*/
import type {
NatsConnection,
JetStreamClient,
JetStreamManager,
Consumer as NatsJsConsumer,
JsMsg,
JetStreamPublishOptions,
} from "nats";
import {
connect,
ErrorCode,
type NatsConnection,
type JetStreamClient,
type JetStreamManager,
type Consumer as NatsJsConsumer,
headers,
type JsMsg,
type JetStreamPublishOptions,
NatsError,
StringCodec,
AckPolicy,
@ -36,7 +38,8 @@ import type {
CreateConsumerOptions,
Message,
} from "./types.js";
import { pubSubError, type PubSubError } from "../errors.js";
import type { PubSubError } from "../errors.js";
import { pubSubError, } from "../errors.js";
const sc = StringCodec();

13
ts/packages/base/src/messaging/consumer.ts
View file

@ -7,20 +7,25 @@
import type { PubSubBackend } from "../backend/types.js";
import { PubSub } from "../backend/pubsub.js";
import type { Flow } from "../processor/flow.js";
import {
import type {
MessagingHandlerError,
MessagingLifecycleError,
} from "../errors.js";
import {
TooManyRequestsError,
messagingHandlerError,
messagingLifecycleError,
type MessagingLifecycleError,
} from "../errors.js";
import { Config as EffectConfig, Effect, Exit, Scope } from "effect";
import type { Config as EffectConfig, } from "effect";
import { Effect, Exit, Scope } from "effect";
import * as P from "effect/Predicate";
import * as S from "effect/Schema";
import { loadMessagingRuntimeConfig } from "../runtime/index.ts";
import type {
EffectConsumer,
} from "./runtime.js";
import {
makeEffectConsumerFromPubSub,
type EffectConsumer,
} from "./runtime.js";
export type MessageHandler<T> = (

9
ts/packages/base/src/messaging/producer.ts
View file

@ -8,11 +8,14 @@ import type { PubSubBackend } from "../backend/types.js";
import type { ProducerMetrics } from "../metrics/index.ts";
import { Effect, Exit, Scope } from "effect";
import { PubSub } from "../backend/pubsub.js";
import { makeEffectProducerFromPubSub, type EffectProducer } from "./runtime.js";
import type { EffectProducer } from "./runtime.js";
import { makeEffectProducerFromPubSub, } from "./runtime.js";
import type {
MessagingDeliveryError,
MessagingLifecycleError,
} from "../errors.js";
import {
messagingLifecycleError,
type MessagingDeliveryError,
type MessagingLifecycleError,
} from "../errors.js";
export interface Producer<T> {

19
ts/packages/base/src/messaging/request-response.ts
View file

@ -7,21 +7,26 @@
* Python reference: trustgraph-base/trustgraph/base/request_response_spec.py
*/
import { Config as EffectConfig, Effect, Exit, Scope } from "effect";
import type { Config as EffectConfig, } from "effect";
import { Effect, Exit, Scope } from "effect";
import type { PubSubBackend } from "../backend/types.js";
import { PubSub } from "../backend/pubsub.js";
import type {
MessagingDeliveryError,
MessagingLifecycleError,
MessagingTimeoutError,
PubSubError,
} from "../errors.js";
import {
messagingLifecycleError,
type MessagingDeliveryError,
type MessagingLifecycleError,
type MessagingTimeoutError,
type PubSubError,
} from "../errors.js";
import { loadMessagingRuntimeConfig } from "../runtime/index.ts";
import type {
EffectRequestOptions,
EffectRequestResponse,
} from "./runtime.js";
import {
makeEffectRequestResponseFromPubSub,
type EffectRequestOptions,
type EffectRequestResponse,
} from "./runtime.js";
export interface RequestResponseOptions {

25
ts/packages/base/src/messaging/runtime.ts
View file

@ -3,6 +3,9 @@
*/
import { randomUUID } from "node:crypto";
import type {
Scope,
} from "effect";
import {
Context,
Deferred,
@ -14,7 +17,6 @@ import {
Ref,
Result,
Schedule,
Scope,
Stream,
} from "effect";
import * as O from "effect/Option";
@ -26,7 +28,16 @@ import type {
CreateProducerOptions,
Message,
} from "../backend/types.js";
import { PubSub, type PubSubService } from "../backend/pubsub.js";
import type { PubSubService } from "../backend/pubsub.js";
import { PubSub, } from "../backend/pubsub.js";
import type {
FlowRuntimeError,
MessagingDeliveryError,
MessagingHandlerError,
MessagingLifecycleError,
MessagingTimeoutError,
PubSubError,
} from "../errors.js";
import {
flowRuntimeError,
messagingDeliveryError,
@ -34,20 +45,16 @@ import {
messagingLifecycleError,
messagingTimeoutError,
TooManyRequestsError,
type FlowRuntimeError,
type MessagingDeliveryError,
type MessagingHandlerError,
type MessagingLifecycleError,
type MessagingTimeoutError,
type PubSubError,
} from "../errors.js";
import type { ProducerMetrics } from "../metrics/index.js";
import type { FlowContext } from "./consumer.js";
import type { Flow } from "../processor/flow.js";
import type { SpecRuntimeRequirements } from "../spec/types.js";
import type {
MessagingRuntimeConfig,
} from "../runtime/messaging-config.js";
import {
loadMessagingRuntimeConfig,
type MessagingRuntimeConfig,
} from "../runtime/messaging-config.js";
const isTooManyRequestsError = S.is(TooManyRequestsError);

3
ts/packages/base/src/metrics/prometheus.ts
View file

@ -4,7 +4,8 @@
* Python reference: trustgraph-base/trustgraph/base/metrics.py
*/
import { Effect, Metric } from "effect";
import type { Effect, } from "effect";
import { Metric } from "effect";
import { PrometheusMetrics } from "effect/unstable/observability";
export const prometheusContentType = "text/plain; version=0.0.4; charset=utf-8";

6
ts/packages/base/src/processor/async-processor.ts
View file

@ -8,8 +8,10 @@
import type { PubSubBackend } from "../backend/types.js";
import { makeNatsBackend, makeNatsBackendScoped } from "../backend/nats.js";
import { Cause, Config as EffectConfig, Context, Effect } from "effect";
import { processorLifecycleError, type ProcessorLifecycleError } from "../errors.js";
import type { Cause, Config as EffectConfig, } from "effect";
import { Context, Effect } from "effect";
import type { ProcessorLifecycleError } from "../errors.js";
import { processorLifecycleError, } from "../errors.js";
import { loadProcessorRuntimeConfig } from "../runtime/config.js";
export interface ProcessorConfig {

24
ts/packages/base/src/processor/flow-processor.ts
View file

@ -7,23 +7,28 @@
* Python reference: trustgraph-base/trustgraph/base/flow_processor.py
*/
import type {
AsyncProcessorRuntime,
EffectConfigHandler,
ProcessorRuntime,
ProcessorConfig,
} from "./async-processor.js";
import {
makeAsyncProcessor,
type AsyncProcessorRuntime,
type EffectConfigHandler,
type ProcessorRuntime,
type ProcessorConfig,
} from "./async-processor.js";
import type { Spec } from "../spec/types.js";
import type { BackendConsumer, PubSubBackend } from "../backend/types.js";
import { Flow, type FlowDefinition } from "./flow.js";
import type { FlowDefinition } from "./flow.js";
import { Flow, } from "./flow.js";
import { topics } from "../schema/topics.js";
import type {
FlowRuntimeError,
ProcessorLifecycleError,
PubSubError,
} from "../errors.js";
import {
errorMessage,
pubSubError,
type FlowRuntimeError,
type ProcessorLifecycleError,
type PubSubError,
} from "../errors.js";
import {
ConsumerFactory,
@ -37,7 +42,8 @@ import {
} from "../messaging/runtime.js";
import { makePubSubService, PubSub } from "../backend/pubsub.js";
import { loadMessagingRuntimeConfig } from "../runtime/index.ts";
import { Config as EffectConfig, Context, Duration, Effect, Exit, Scope } from "effect";
import type { Config as EffectConfig, Context, } from "effect";
import { Duration, Effect, Exit, Scope } from "effect";
import * as MutableHashMap from "effect/MutableHashMap";
import * as O from "effect/Option";
import * as S from "effect/Schema";

27
ts/packages/base/src/processor/flow.ts
View file

@ -4,30 +4,35 @@
* Python reference: trustgraph-base/trustgraph/base/flow.py
*/
import { Config as EffectConfig, Context, Effect, Exit, Scope } from "effect";
import type { Config as EffectConfig, Context, } from "effect";
import { Effect, Exit, Scope } from "effect";
import * as MutableHashMap from "effect/MutableHashMap";
import * as O from "effect/Option";
import * as S from "effect/Schema";
import type { PubSubBackend } from "../backend/types.js";
import { makePubSubService } from "../backend/pubsub.js";
import type {
FlowParameterDecodeError,
FlowResourceNotFoundError,
MessagingDeliveryError,
MessagingLifecycleError,
MessagingTimeoutError,
PubSubError,
} from "../errors.js";
import {
flowParameterDecodeError,
flowResourceNotFoundError,
type FlowParameterDecodeError,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type MessagingLifecycleError,
type MessagingTimeoutError,
type PubSubError,
} from "../errors.js";
import type {
EffectConsumer,
EffectProducer,
EffectRequestOptions,
EffectRequestResponse,
} from "../messaging/runtime.js";
import {
ConsumerFactory,
ProducerFactory,
RequestResponseFactory,
type EffectConsumer,
type EffectProducer,
type EffectRequestOptions,
type EffectRequestResponse,
makeConsumerFactoryService,
makeProducerFactoryService,
makeRequestResponseFactoryService,

15
ts/packages/base/src/processor/program.ts
View file

@ -5,11 +5,12 @@
* executable path while the processor internals remain Promise-based.
*/
import { Config as EffectConfig, Effect, Layer } from "effect";
import {
type FlowRuntimeError,
type ProcessorLifecycleError,
type PubSubError,
import type { Config as EffectConfig, } from "effect";
import { Effect, Layer } from "effect";
import type {
FlowRuntimeError,
ProcessorLifecycleError,
PubSubError,
} from "../errors.js";
import { makeNatsBackendScoped } from "../backend/nats.js";
import { makePubSubService, PubSub } from "../backend/pubsub.js";
@ -23,9 +24,11 @@ import {
makeRequestResponseFactoryService,
runFlowRuntimeScoped,
} from "../messaging/runtime.js";
import type {
ProcessorRuntimeConfigOptions,
} from "../runtime/config.js";
import {
loadProcessorRuntimeConfig,
type ProcessorRuntimeConfigOptions,
} from "../runtime/config.js";
import { loadMessagingRuntimeConfig } from "../runtime/messaging-config.js";
import type {

8
ts/packages/base/src/services/embeddings-service.ts
View file

@ -5,11 +5,13 @@
*/
import { Context, Effect } from "effect";
import type {
EmbeddingsError,
FlowResourceNotFoundError,
MessagingDeliveryError,
} from "../errors.js";
import {
errorMessage,
type EmbeddingsError,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
} from "../errors.js";
import type { FlowContext } from "../messaging/consumer.js";
import { makeFlowProcessor } from "../processor/index.ts";

6
ts/packages/base/src/services/llm-service.ts
View file

@ -6,10 +6,12 @@
import { Context, Effect, Stream } from "effect";
import * as S from "effect/Schema";
import type {
FlowResourceNotFoundError,
MessagingDeliveryError,
} from "../errors.js";
import {
errorMessage,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
} from "../errors.js";
import type { FlowContext } from "../messaging/consumer.js";
import { makeFlowProcessor } from "../processor/index.ts";

10
ts/packages/base/src/spec/consumer-spec.ts
View file

@ -8,12 +8,14 @@ import { Effect } from "effect";
import type { Spec } from "./types.js";
import type { SpecRuntimeRequirements } from "./types.js";
import type { Flow, FlowDefinition } from "../processor/flow.js";
import {
ConsumerFactory,
type EffectMessageHandler,
import type {
EffectMessageHandler,
} from "../messaging/runtime.js";
import {
type PubSubError,
ConsumerFactory,
} from "../messaging/runtime.js";
import type {
PubSubError,
} from "../errors.js";
declare const ConsumerSpecType: unique symbol;

14
ts/packages/base/src/spec/producer-spec.ts
View file

@ -7,13 +7,17 @@
import { Effect } from "effect";
import type { SpecRuntimeRequirements } from "./types.js";
import type { Flow, FlowDefinition } from "../processor/flow.js";
import {
flowResourceNotFoundError,
type FlowResourceNotFoundError,
type PubSubError,
import type {
FlowResourceNotFoundError,
PubSubError,
} from "../errors.js";
import {
type EffectProducer,
flowResourceNotFoundError,
} from "../errors.js";
import type {
EffectProducer,
} from "../messaging/runtime.js";
import {
ProducerFactory,
} from "../messaging/runtime.js";

14
ts/packages/base/src/spec/request-response-spec.ts
View file

@ -10,13 +10,17 @@
import { Effect } from "effect";
import type { SpecRuntimeRequirements } from "./types.js";
import type { Flow, FlowDefinition } from "../processor/flow.js";
import {
flowResourceNotFoundError,
type FlowResourceNotFoundError,
type PubSubError,
import type {
FlowResourceNotFoundError,
PubSubError,
} from "../errors.js";
import {
type EffectRequestResponse,
flowResourceNotFoundError,
} from "../errors.js";
import type {
EffectRequestResponse,
} from "../messaging/runtime.js";
import {
RequestResponseFactory,
} from "../messaging/runtime.js";

3
ts/packages/cli/package.json
View file

@ -9,7 +9,8 @@
"build": "bunx --bun tsc",
"dev": "tsc --watch",
"clean": "rm -rf dist",
"test": "bunx --bun vitest run --passWithNoTests --exclude=dist/**"
"test": "bunx --bun vitest run --passWithNoTests --exclude=dist/**",
"lint": "bunx --bun biome check src"
},
"dependencies": {
"@effect/platform-bun": "4.0.0-beta.78",

3
ts/packages/cli/src/commands/agent.ts
View file

@ -7,7 +7,8 @@
import { Effect } from "effect";
import * as Argument from "effect/unstable/cli/Argument";
import * as Command from "effect/unstable/cli/Command";
import { cliCommandError, withGatewayClient, type CliCommandError } from "./util.js";
import type { CliCommandError } from "./util.js";
import { cliCommandError, withGatewayClient, } from "./util.js";
function asRecord(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)

6
ts/packages/cli/src/commands/util.ts
View file

@ -2,11 +2,13 @@
* Shared CLI utilities.
*/
import type {
BaseApi,
TrustGraphGatewayClient,
} from "@trustgraph/client";
import {
createTrustGraphSocket,
makeTrustGraphGatewayClientScoped,
type BaseApi,
type TrustGraphGatewayClient,
} from "@trustgraph/client";
import { Duration, Effect } from "effect";
import * as O from "effect/Option";

4
ts/packages/client/package.json
View file

@ -19,7 +19,8 @@
"build": "bunx --bun tsc",
"dev": "tsc --watch",
"clean": "rm -rf dist",
"test": "bunx --bun vitest run"
"test": "bunx --bun vitest run",
"lint": "bunx --bun biome check src"
},
"dependencies": {
"effect": "4.0.0-beta.78"
@ -36,7 +37,6 @@
"@effect/vitest": "4.0.0-beta.78",
"@types/node": "^22.0.0",
"@types/ws": "^8.5.0",
"typescript": "^5.8.0",
"vitest": "^4.1.6",
"happy-dom": "^20.0.0"

6
ts/packages/client/src/__tests__/flows-api.test.ts
View file

@ -1,6 +1,7 @@
import { describe, it, expect, vi, beforeEach } from "vitest";
import type { BaseApi } from "../socket/trustgraph-socket";
import { FlowsApi, TrustGraphSocketError } from "../socket/trustgraph-socket";
import { FlowResponse } from "../models/messages";
import type { FlowResponse } from "../models/messages";
describe("FlowsApi", () => {
let mockApi: {
@ -12,8 +13,7 @@ describe("FlowsApi", () => {
mockApi = {
makeRequest: vi.fn(),
};
// eslint-disable-next-line @typescript-eslint/no-explicit-any
flowsApi = FlowsApi(mockApi as any);
flowsApi = FlowsApi(mockApi as unknown as BaseApi);
});
describe("startFlow", () => {

2
ts/packages/client/src/__tests__/messages.test.ts
View file

@ -342,7 +342,7 @@ describe("Message Types", () => {
type: "library-error",
});
expect(response["document-metadatas"]).toHaveLength(1);
expect(response["document-metadatas"]![0].id).toBe("doc-1");
expect(response["document-metadatas"]?.[0].id).toBe("doc-1");
});
});

6
ts/packages/client/src/__tests__/rpc-timeout.test.ts
View file

@ -1,8 +1,10 @@
import { Effect } from "effect";
import { describe, expect, it, vi } from "vitest";
import { DispatchError, DispatchStreamChunk } from "../rpc/contract";
import { type DispatchInput, type RpcConnectionState, withDispatchRequestPolicy } from "../socket/effect-rpc-client";
import { type ConnectionState, makeBaseApiWithRpc } from "../socket/trustgraph-socket";
import type { DispatchInput, RpcConnectionState, } from "../socket/effect-rpc-client";
import { withDispatchRequestPolicy } from "../socket/effect-rpc-client";
import type { ConnectionState, } from "../socket/trustgraph-socket";
import { makeBaseApiWithRpc } from "../socket/trustgraph-socket";
const input: DispatchInput = {
scope: "global",

4
ts/packages/client/src/__tests__/workbench-contracts.test.ts
View file

@ -1,6 +1,8 @@
import { describe, expect, it, vi } from "vitest";
import {
import type {
BaseApi,
} from "../socket/trustgraph-socket";
import {
ConfigApi,
KnowledgeApi,
LibrarianApi,

40
ts/packages/client/src/models/namespaces.ts
View file

@ -5,38 +5,38 @@
// TrustGraph namespace
export const TG = "https://trustgraph.ai/ns/";
export const TG_QUERY = TG + "query";
export const TG_EDGE_COUNT = TG + "edgeCount";
export const TG_SELECTED_EDGE = TG + "selectedEdge";
export const TG_EDGE = TG + "edge";
export const TG_REASONING = TG + "reasoning";
export const TG_CONTENT = TG + "content";
export const TG_REIFIES = TG + "reifies";
export const TG_DOCUMENT = TG + "document";
export const TG_QUERY = `${TG}query`;
export const TG_EDGE_COUNT = `${TG}edgeCount`;
export const TG_SELECTED_EDGE = `${TG}selectedEdge`;
export const TG_EDGE = `${TG}edge`;
export const TG_REASONING = `${TG}reasoning`;
export const TG_CONTENT = `${TG}content`;
export const TG_REIFIES = `${TG}reifies`;
export const TG_DOCUMENT = `${TG}document`;
// W3C PROV-O namespace
export const PROV = "http://www.w3.org/ns/prov#";
export const PROV_STARTED_AT_TIME = PROV + "startedAtTime";
export const PROV_WAS_DERIVED_FROM = PROV + "wasDerivedFrom";
export const PROV_WAS_GENERATED_BY = PROV + "wasGeneratedBy";
export const PROV_ACTIVITY = PROV + "Activity";
export const PROV_ENTITY = PROV + "Entity";
export const PROV_STARTED_AT_TIME = `${PROV}startedAtTime`;
export const PROV_WAS_DERIVED_FROM = `${PROV}wasDerivedFrom`;
export const PROV_WAS_GENERATED_BY = `${PROV}wasGeneratedBy`;
export const PROV_ACTIVITY = `${PROV}Activity`;
export const PROV_ENTITY = `${PROV}Entity`;
// RDFS namespace
export const RDFS = "http://www.w3.org/2000/01/rdf-schema#";
export const RDFS_LABEL = RDFS + "label";
export const RDFS_LABEL = `${RDFS}label`;
// RDF namespace
export const RDF = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";
export const RDF_TYPE = RDF + "type";
export const RDF_TYPE = `${RDF}type`;
// Schema.org namespace (used in document metadata)
export const SCHEMA = "https://schema.org/";
export const SCHEMA_NAME = SCHEMA + "name";
export const SCHEMA_DESCRIPTION = SCHEMA + "description";
export const SCHEMA_AUTHOR = SCHEMA + "author";
export const SCHEMA_KEYWORDS = SCHEMA + "keywords";
export const SCHEMA_NAME = `${SCHEMA}name`;
export const SCHEMA_DESCRIPTION = `${SCHEMA}description`;
export const SCHEMA_AUTHOR = `${SCHEMA}author`;
export const SCHEMA_KEYWORDS = `${SCHEMA}keywords`;
// SKOS namespace
export const SKOS = "http://www.w3.org/2004/02/skos/core#";
export const SKOS_DEFINITION = SKOS + "definition";
export const SKOS_DEFINITION = `${SKOS}definition`;

3
ts/packages/client/src/socket/effect-rpc-client.ts
View file

@ -4,7 +4,8 @@ import * as RpcClient from "effect/unstable/rpc/RpcClient";
import type { RpcClientError } from "effect/unstable/rpc/RpcClientError";
import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization";
import * as Socket from "effect/unstable/socket/Socket";
import { DispatchPayload, DispatchError, TrustGraphRpcs, type DispatchStreamChunk } from "../rpc/contract.js";
import type { DispatchStreamChunk } from "../rpc/contract.js";
import { DispatchPayload, DispatchError, TrustGraphRpcs, } from "../rpc/contract.js";
type TrustGraphRpcClient = RpcClient.RpcClient<
RpcGroup.Rpcs<typeof TrustGraphRpcs>,

17
ts/packages/client/src/socket/trustgraph-socket.ts
View file

@ -1,10 +1,12 @@
// Import core types and classes for the TrustGraph API
import type { Term, Triple } from "../models/Triple.js";
import type {
EffectRpcClient,
DispatchInput,
DispatchOptions,
RpcConnectionState,
} from "./effect-rpc-client.js";
import {
type EffectRpcClient,
type DispatchInput,
type DispatchOptions,
type RpcConnectionState,
makeEffectRpcClient,
} from "./effect-rpc-client.js";
import { getDefaultSocketUrl, getRandomValues } from "./websocket-adapter.js";
@ -479,7 +481,7 @@ export function makeBaseApi(
hasApiKey: isNonEmptyString(token),
}),
);
let lastError: string | undefined = undefined;
let lastError: string | undefined ;
let rpcState: RpcConnectionState = { status: "connecting" };
const api = {
@ -536,7 +538,7 @@ export function makeBaseApi(
* Format: {clientTag}-{incrementingNumber}
*/
getNextId() {
const mid = api.tag + "-" + api.id.toString();
const mid = `${api.tag}-${api.id.toString()}`;
api.id++;
return mid;
},
@ -2530,11 +2532,10 @@ export function makeKnowledgeApi(api: BaseApi) {
// End of stream - notify receiver and signal completion
receiver(msg, true);
return true;
} else {
}
// Regular message - continue streaming
receiver(msg, false);
return false;
}
};
return this.api.makeRequestMulti<LibraryRequest, LibraryResponse>(

3
ts/packages/client/src/socket/websocket-adapter.ts
View file

@ -56,8 +56,7 @@ export interface IsomorphicWebSocket {
addEventListener(type: "close", listener: (event: WsCloseEvent) => void): void;
addEventListener(type: "open", listener: (event: WsEvent) => void): void;
addEventListener(type: "error", listener: (event: WsEvent) => void): void;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
removeEventListener(type: string, listener: (...args: any[]) => void): void;
removeEventListener(type: string, listener: (...args: ReadonlyArray<never>) => void): void;
}
/** Constructor signature for an isomorphic WebSocket implementation. */

3
ts/packages/flow/package.json
View file

@ -8,7 +8,8 @@
"build": "bunx --bun tsc",
"dev": "tsc --watch",
"clean": "rm -rf dist",
"test": "bunx --bun vitest run"
"test": "bunx --bun vitest run",
"lint": "bunx --bun biome check src"
},
"dependencies": {
"@effect/ai-anthropic": "4.0.0-beta.78",

18
ts/packages/flow/src/__tests__/chunking-service.test.ts
View file

@ -2,19 +2,21 @@ import { describe, expect, it } from "@effect/vitest";
import { ConfigProvider, Effect, Fiber } from "effect";
import * as EffectChunk from "effect/Chunk";
import * as S from "effect/Schema";
import type {
BackendConsumer,
BackendProducer,
Chunk,
CreateConsumerOptions,
CreateProducerOptions,
Message,
PubSubBackend,
TextDocument,
} from "@trustgraph/base";
import {
MessagingRuntimeLive,
PubSub,
runProcessorScoped,
topics,
type BackendConsumer,
type BackendProducer,
type Chunk,
type CreateConsumerOptions,
type CreateProducerOptions,
type Message,
type PubSubBackend,
type TextDocument,
} from "@trustgraph/base";
import { ChunkingService } from "../chunking/service.js";
import { recursiveSplit } from "../chunking/recursive-splitter.js";

12
ts/packages/flow/src/__tests__/falkordb-lifecycle.test.ts
View file

@ -1,16 +1,20 @@
import { Effect } from "effect";
import { describe, expect, it } from "vitest";
import type {
FalkorDBClosableClient as FalkorDBQueryClient,
FalkorDBQueryGraph,
} from "../query/triples/falkordb.js";
import {
FalkorDBTriplesQueryLive,
FalkorDBTriplesQueryService,
type FalkorDBClosableClient as FalkorDBQueryClient,
type FalkorDBQueryGraph,
} from "../query/triples/falkordb.js";
import type {
FalkorDBClosableClient as FalkorDBStoreClient,
FalkorDBStoreGraph,
} from "../storage/triples/falkordb.js";
import {
FalkorDBTriplesStoreLive,
FalkorDBTriplesStoreService,
type FalkorDBClosableClient as FalkorDBStoreClient,
type FalkorDBStoreGraph,
} from "../storage/triples/falkordb.js";
class FakeFalkorDBClient implements FalkorDBStoreClient, FalkorDBQueryClient {

24
ts/packages/flow/src/__tests__/flow-manager-service.test.ts
View file

@ -1,18 +1,20 @@
import {Effect, HashMap, Option, SynchronizedRef} from "effect";
import {describe, expect, it} from "vitest";
import type {
BackendConsumer,
BackendProducer,
ConfigRequest,
ConfigResponse,
CreateConsumerOptions,
CreateProducerOptions,
FlowRequest,
FlowResponse,
Message,
PubSubBackend,
RequestResponse,
} from "@trustgraph/base";
import {
topics,
type BackendConsumer,
type BackendProducer,
type ConfigRequest,
type ConfigResponse,
type CreateConsumerOptions,
type CreateProducerOptions,
type FlowRequest,
type FlowResponse,
type Message,
type PubSubBackend,
type RequestResponse,
} from "@trustgraph/base";
import {FlowManagerError, makeFlowManagerService} from "../flow-manager/service.js";

3
ts/packages/flow/src/__tests__/gateway-routes.test.ts
View file

@ -3,7 +3,8 @@ import { Effect, Exit, Scope } from "effect";
import { HttpRouter, HttpServerResponse } from "effect/unstable/http";
import type { DispatcherManager } from "../gateway/dispatch/manager.js";
import type { GatewayRpcServer } from "../gateway/rpc-server.js";
import { makeGatewayRoutes, type GatewayConfig } from "../gateway/server.js";
import type { GatewayConfig } from "../gateway/server.js";
import { makeGatewayRoutes, } from "../gateway/server.js";
interface DispatchCall {
readonly scope: "global" | "flow";

20
ts/packages/flow/src/__tests__/knowledge-core-service.test.ts
View file

@ -3,17 +3,19 @@ import {tmpdir} from "node:os";
import {join} from "node:path";
import {Effect, HashMap, Option, SynchronizedRef} from "effect";
import {describe, expect, it} from "vitest";
import type {
BackendConsumer,
BackendProducer,
CreateConsumerOptions,
CreateProducerOptions,
KnowledgeRequest,
KnowledgeResponse,
Message,
PubSubBackend,
Triple,
} from "@trustgraph/base";
import {
topics,
type BackendConsumer,
type BackendProducer,
type CreateConsumerOptions,
type CreateProducerOptions,
type KnowledgeRequest,
type KnowledgeResponse,
type Message,
type PubSubBackend,
type Triple,
} from "@trustgraph/base";
import {makeKnowledgeCoreService} from "../cores/service.js";

18
ts/packages/flow/src/__tests__/librarian-service.test.ts
View file

@ -3,15 +3,15 @@ import {tmpdir} from "node:os";
import {join} from "node:path";
import {Effect} from "effect";
import {describe, expect, it} from "vitest";
import {
type BackendConsumer,
type BackendProducer,
type CreateConsumerOptions,
type CreateProducerOptions,
type DocumentMetadata,
type Message,
type PubSubBackend,
type Triple,
import type {
BackendConsumer,
BackendProducer,
CreateConsumerOptions,
CreateProducerOptions,
DocumentMetadata,
Message,
PubSubBackend,
Triple,
} from "@trustgraph/base";
import {makeLibrarianService} from "../librarian/service.js";

18
ts/packages/flow/src/__tests__/prompt-template.test.ts
View file

@ -1,19 +1,21 @@
import { describe, expect, it } from "@effect/vitest";
import { ConfigProvider, Effect, Fiber } from "effect";
import * as S from "effect/Schema";
import type {
BackendConsumer,
BackendProducer,
CreateConsumerOptions,
CreateProducerOptions,
Message,
PromptRequest,
PromptResponse,
PubSubBackend,
} from "@trustgraph/base";
import {
MessagingRuntimeLive,
PubSub,
runProcessorScoped,
topics,
type BackendConsumer,
type BackendProducer,
type CreateConsumerOptions,
type CreateProducerOptions,
type Message,
type PromptRequest,
type PromptResponse,
type PubSubBackend,
} from "@trustgraph/base";
import { PromptTemplateService } from "../prompt/template.js";

6
ts/packages/flow/src/__tests__/retrieval-rag.test.ts
View file

@ -15,8 +15,10 @@ import type {
TriplesQueryRequest,
TriplesQueryResponse,
} from "@trustgraph/base";
import { makeDocumentRagEngine, type DocumentRagClients } from "../retrieval/document-rag.js";
import { makeGraphRagEngine, type GraphRagClients } from "../retrieval/graph-rag.js";
import type { DocumentRagClients } from "../retrieval/document-rag.js";
import { makeDocumentRagEngine, } from "../retrieval/document-rag.js";
import type { GraphRagClients } from "../retrieval/graph-rag.js";
import { makeGraphRagEngine, } from "../retrieval/graph-rag.js";
const requestor = <TReq, TRes>(
handler: (request: TReq) => TRes,

21
ts/packages/flow/src/agent/mcp-tool/service.ts
View file

@ -14,22 +14,23 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
import type { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
import { NodeRuntime } from "@effect/platform-node";
import type {
ProcessorConfig,
FlowContext,
FlowProcessorRuntime,
ToolRequest,
ToolResponse,
EffectConfigHandler,
FlowResourceNotFoundError,
MessagingDeliveryError,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
makeProducerSpec,
makeFlowProcessorProgram,
errorMessage,
type ProcessorConfig,
type FlowContext,
type FlowProcessorRuntime,
type ToolRequest,
type ToolResponse,
type EffectConfigHandler,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type Spec,
} from "@trustgraph/base";
import { Context, Effect, Layer, Ref } from "effect";
import * as O from "effect/Option";

55
ts/packages/flow/src/agent/react/service.ts
View file

@ -19,6 +19,27 @@
import {
NodeRuntime,
} from "@effect/platform-node";
import type {
ProcessorConfig,
FlowContext,
FlowProcessorRuntime,
AgentRequest,
AgentResponse,
TextCompletionRequest,
TextCompletionResponse,
GraphRagRequest,
GraphRagResponse,
DocumentRagRequest,
DocumentRagResponse,
TriplesQueryRequest,
TriplesQueryResponse,
ToolRequest,
ToolResponse,
EffectConfigHandler,
FlowResourceNotFoundError,
MessagingDeliveryError,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
@ -26,37 +47,19 @@ import {
makeRequestResponseSpec,
makeFlowProcessorProgram,
errorMessage,
type ProcessorConfig,
type FlowContext,
type FlowProcessorRuntime,
type AgentRequest,
type AgentResponse,
type TextCompletionRequest,
type TextCompletionResponse,
type GraphRagRequest,
type GraphRagResponse,
type DocumentRagRequest,
type DocumentRagResponse,
type TriplesQueryRequest,
type TriplesQueryResponse,
type ToolRequest,
type ToolResponse,
type EffectConfigHandler,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type Spec,
} from "@trustgraph/base";
import {Context, Effect, Layer, Match, Ref} from "effect";
import * as O from "effect/Option";
import * as Predicate from "effect/Predicate";
import * as S from "effect/Schema";
import type {
ExplainData,
} from "./tools.js";
import {
createKnowledgeQueryTool,
createDocumentQueryTool,
createTriplesQueryTool,
createMcpTool,
type ExplainData,
} from "./tools.js";
import { buildReActPrompt } from "./prompt.js";
import { filterToolsByGroupAndState } from "../tool-filter.js";
@ -550,9 +553,9 @@ export function parseReActResponse(text: string): {
const firstLine = trimmed.slice("Final Answer:".length).trim();
const remainingLines = lines.slice(i + 1).join("\n").trim();
finalAnswer =
firstLine + (remainingLines.length > 0 ? "\n" + remainingLines : "");
firstLine + (remainingLines.length > 0 ? `\n${remainingLines}` : "");
break;
} else if (trimmed.startsWith("Thought:")) {
}if (trimmed.startsWith("Thought:")) {
currentSection = "thought";
const content = trimmed.slice("Thought:".length).trim();
if (content.length > 0) {
@ -577,14 +580,14 @@ export function parseReActResponse(text: string): {
// Continuation line for current section
Match.value(currentSection).pipe(
Match.when("thought", () => {
thought += "\n" + trimmed;
thought += `\n${trimmed}`;
}),
Match.when("action", () => {
// Action should be a single line (tool name), but handle multi-line
action += " " + trimmed;
action += ` ${trimmed}`;
}),
Match.when("action_input", () => {
actionInput += "\n" + trimmed;
actionInput += `\n${trimmed}`;
}),
Match.exhaustive,
);

4
ts/packages/flow/src/agent/react/tools.ts
View file

@ -23,8 +23,8 @@ import { Effect, Match } from "effect";
import * as O from "effect/Option";
import * as Predicate from "effect/Predicate";
import * as S from "effect/Schema";
import { agentToolError, type AgentTool, type ToolArg } from "./types.js";
import type { AgentTool, ToolArg } from "./types.js";
import { agentToolError, } from "./types.js";
const decodeJsonUnknown = S.decodeUnknownOption(S.UnknownFromJsonString);
const decodeTerm = S.decodeUnknownOption(TermSchema);

4
ts/packages/flow/src/agent/tool-filter.ts
View file

@ -30,7 +30,7 @@ function isToolAvailable(
const config = tool.config ?? {};
// Get tool groups (default to ["default"])
let toolGroups = config["group"] as string[] | string | undefined;
let toolGroups = config.group as string[] | string | undefined;
if (toolGroups === undefined) toolGroups = ["default"];
if (!Array.isArray(toolGroups)) toolGroups = [toolGroups];
@ -56,6 +56,6 @@ function isToolAvailable(
* Get the next state after successful tool execution.
*/
export function getNextState(tool: AgentTool, currentState: string): string {
const nextState = tool.config?.["state"] as string | undefined;
const nextState = tool.config?.state as string | undefined;
return nextState ?? currentState;
}

20
ts/packages/flow/src/chunking/service.ts
View file

@ -9,20 +9,22 @@
* Python reference: trustgraph-flow/trustgraph/chunking/recursive_splitter/service.py
*/
import type {
ProcessorConfig,
FlowProcessorRuntime,
FlowContext,
FlowResourceNotFoundError,
MessagingDeliveryError,
TextDocument,
Chunk,
Triples,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
makeProducerSpec,
makeParameterSpec,
type ProcessorConfig,
type FlowProcessorRuntime,
type FlowContext,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type TextDocument,
type Chunk,
type Triples,
type Spec,
} from "@trustgraph/base";
import { NodeRuntime } from "@effect/platform-node";
import { makeFlowProcessorProgram } from "@trustgraph/base";

18
ts/packages/flow/src/config/service.ts
View file

@ -8,6 +8,16 @@ import {NodeRuntime} from "@effect/platform-node";
import {Duration, Effect, HashMap, Match, Option, SynchronizedRef} from "effect";
import * as Predicate from "effect/Predicate";
import * as S from "effect/Schema";
import type {
AsyncProcessorRuntime,
BackendConsumer,
BackendProducer,
ConfigOperation,
ConfigRequest,
ConfigResponse,
Message,
ProcessorConfig,
} from "@trustgraph/base";
import {
ConfigRequest as ConfigRequestSchema,
ConfigResponse as ConfigResponseSchema,
@ -18,14 +28,6 @@ import {
optionalStringConfig,
processorLifecycleError,
topics,
type AsyncProcessorRuntime,
type BackendConsumer,
type BackendProducer,
type ConfigOperation,
type ConfigRequest,
type ConfigResponse,
type Message,
type ProcessorConfig,
} from "@trustgraph/base";
import {readTextFileEffect, writeTextFileEffect} from "../runtime/effect-files.js";

20
ts/packages/flow/src/cores/service.ts
View file

@ -5,6 +5,17 @@
*/
import {NodeRuntime} from "@effect/platform-node";
import type {
AsyncProcessorRuntime,
BackendConsumer,
BackendProducer,
KnowledgeOperation,
KnowledgeRequest,
KnowledgeResponse,
Message,
ProcessorConfig,
PubSubError,
} from "@trustgraph/base";
import {
KnowledgeRequest as KnowledgeRequestSchema,
KnowledgeResponse as KnowledgeResponseSchema,
@ -17,15 +28,6 @@ import {
optionalStringConfig,
processorLifecycleError,
topics,
type AsyncProcessorRuntime,
type BackendConsumer,
type BackendProducer,
type KnowledgeOperation,
type KnowledgeRequest,
type KnowledgeResponse,
type Message,
type ProcessorConfig,
type PubSubError,
} from "@trustgraph/base";
import {Duration, Effect, HashMap, Match, SynchronizedRef} from "effect";
import * as O from "effect/Option";

32
ts/packages/flow/src/decoding/pdf-decoder.ts
View file

@ -15,26 +15,28 @@
import { getDocument } from "pdfjs-dist/legacy/build/pdf.mjs";
import type { TextItem } from "pdfjs-dist/types/src/display/api.js";
import type {
ProcessorConfig,
FlowProcessorRuntime,
FlowContext,
FlowResourceNotFoundError,
Document,
TextDocument,
Triples,
Triple,
Term,
LibrarianRequest,
LibrarianResponse,
MessagingDeliveryError,
MessagingLifecycleError,
MessagingTimeoutError,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
makeProducerSpec,
makeRequestResponseSpec,
type ProcessorConfig,
type FlowProcessorRuntime,
type FlowContext,
type FlowResourceNotFoundError,
type Document,
type TextDocument,
type Triples,
type Triple,
type Term,
type LibrarianRequest,
type LibrarianResponse,
type MessagingDeliveryError,
type MessagingLifecycleError,
type MessagingTimeoutError,
type Spec,
errorMessage,
} from "@trustgraph/base";
import { NodeRuntime } from "@effect/platform-node";

6
ts/packages/flow/src/embeddings/ollama.ts
View file

@ -8,14 +8,16 @@ import { NodeRuntime } from "@effect/platform-node";
import { Config, Effect, Layer } from "effect";
import * as O from "effect/Option";
import * as S from "effect/Schema";
import type {
EmbeddingsServiceShape,
ProcessorConfig,
} from "@trustgraph/base";
import {
Embeddings,
EmbeddingsError,
errorMessage,
makeEmbeddingsService,
makeEmbeddingsSpecs,
type EmbeddingsServiceShape,
type ProcessorConfig,
} from "@trustgraph/base";
import { makeFlowProcessorProgram } from "@trustgraph/base";

38
ts/packages/flow/src/extract/knowledge-extract.ts
View file

@ -10,29 +10,31 @@
* Python reference: trustgraph-flow/trustgraph/extract/knowledge/service.py
*/
import type {
ProcessorConfig,
FlowProcessorRuntime,
FlowContext,
Chunk,
Triples,
EntityContexts,
EntityContext,
PromptRequest,
PromptResponse,
TextCompletionRequest,
TextCompletionResponse,
Triple,
Term,
FlowResourceNotFoundError,
MessagingDeliveryError,
EffectRequestResponse,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
makeProducerSpec,
makeRequestResponseSpec,
makeFlowProcessorProgram,
type ProcessorConfig,
type FlowProcessorRuntime,
type FlowContext,
type Chunk,
type Triples,
type EntityContexts,
type EntityContext,
type PromptRequest,
type PromptResponse,
type TextCompletionRequest,
type TextCompletionResponse,
type Triple,
type Term,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type EffectRequestResponse,
type Spec,
} from "@trustgraph/base";
import { NodeRuntime } from "@effect/platform-node";
import { Effect } from "effect";
@ -375,7 +377,7 @@ function jsonCandidates(raw: string): ReadonlyArray<string> {
const partial = arrayMatch[0];
const lastBrace = partial.lastIndexOf("}");
if (lastBrace > 0) {
candidates.push(partial.slice(0, lastBrace + 1) + "]");
candidates.push(`${partial.slice(0, lastBrace + 1)}]`);
}
}

20
ts/packages/flow/src/flow-manager/service.ts
View file

@ -14,21 +14,23 @@
* Python reference: trustgraph-flow/trustgraph/flow/service.py
*/
import type {
ProcessorConfig,
AsyncProcessorRuntime,
BackendConsumer,
BackendProducer,
RequestResponse,
ConfigRequest,
ConfigResponse,
FlowRequest,
FlowResponse,
} from "@trustgraph/base";
import {
makeAsyncProcessor,
type ProcessorConfig,
type AsyncProcessorRuntime,
type BackendConsumer,
type BackendProducer,
topics,
makeRequestResponse,
type RequestResponse,
type ConfigRequest,
type ConfigResponse,
FlowRequest as FlowRequestSchema,
FlowResponse as FlowResponseSchema,
type FlowRequest,
type FlowResponse,
errorMessage,
processorLifecycleError,
} from "@trustgraph/base";

20
ts/packages/flow/src/gateway/dispatch/manager.ts
View file

@ -9,6 +9,15 @@
*/
import { Clock, Effect, Exit, HashMap, HashSet, Option, Random, Scope, SynchronizedRef, Tuple } from "effect";
import type {
EffectRequestResponse,
MessagingDeliveryError,
MessagingLifecycleError,
MessagingTimeoutError,
PubSubBackend,
PubSubError,
RequestResponseFactoryService,
} from "@trustgraph/base";
import {
loadMessagingRuntimeConfig,
makeNatsBackend,
@ -17,19 +26,14 @@ import {
makeRequestResponseFactoryService,
messagingDeliveryError,
messagingLifecycleError,
type EffectRequestResponse,
type MessagingDeliveryError,
type MessagingLifecycleError,
type MessagingTimeoutError,
type PubSubBackend,
type PubSubError,
type RequestResponseFactoryService,
} from "@trustgraph/base";
import type { GatewayConfig } from "../server.js";
import type {
DispatchSerializationError,
} from "./serialize.js";
import {
translateRequestEffect,
translateResponseEffect,
type DispatchSerializationError,
} from "./serialize.js";
export type EffectResponder<E = never, R = never> = (

6
ts/packages/flow/src/gateway/dispatch/serialize.ts
View file

@ -18,6 +18,10 @@
* Python reference: trustgraph-base/trustgraph/messaging/translators/primitives.py
*/
import type {
Term,
Triple,
} from "@trustgraph/base";
import {
BlankTerm,
errorMessage,
@ -26,8 +30,6 @@ import {
Term as TermSchema,
Triple as TripleSchema,
TripleTerm,
type Term,
type Triple,
} from "@trustgraph/base";
import { Effect, HashSet, Match } from "effect";
import * as O from "effect/Option";

8
ts/packages/flow/src/gateway/rpc-server.ts
View file

@ -1,10 +1,12 @@
import { Cause, Effect, Layer, Queue, Scope } from "effect";
import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
import type { Cause, Scope } from "effect";
import { Effect, Layer, Queue, } from "effect";
import type { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization";
import * as RpcServer from "effect/unstable/rpc/RpcServer";
import { errorMessage } from "@trustgraph/base";
import type { DispatcherManager, DispatcherStreamError } from "./dispatch/manager.js";
import { DispatchError, DispatchPayload, DispatchStreamChunk, TrustGraphRpcs } from "./rpc-contract.js";
import type { DispatchPayload, } from "./rpc-contract.js";
import { DispatchError, DispatchStreamChunk, TrustGraphRpcs } from "./rpc-contract.js";
export interface GatewayRpcServer {
readonly httpEffect: Effect.Effect<

10
ts/packages/flow/src/gateway/server.ts
View file

@ -12,17 +12,21 @@ import * as O from "effect/Option";
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
import { HttpApiBuilder } from "effect/unstable/httpapi";
import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization";
import type {
PubSubBackend,
} from "@trustgraph/base";
import {
formatPrometheusMetrics,
messagingLifecycleError,
optionalStringConfig,
prometheusContentType,
toTgError,
type PubSubBackend,
} from "@trustgraph/base";
import { GatewayWorkbenchHttpApi } from "@trustgraph/client/rpc/contract";
import { makeDispatcherManagerScoped, type DispatcherManager } from "./dispatch/manager.js";
import { makeGatewayRpcServer, type GatewayRpcServer } from "./rpc-server.js";
import type { DispatcherManager } from "./dispatch/manager.js";
import { makeDispatcherManagerScoped, } from "./dispatch/manager.js";
import type { GatewayRpcServer } from "./rpc-server.js";
import { makeGatewayRpcServer, } from "./rpc-server.js";
export interface GatewayConfig {
port: number;

66
ts/packages/flow/src/librarian/service.ts
View file

@ -10,32 +10,36 @@
* Python reference: trustgraph-flow/trustgraph/librarian/service/service.py
*/
import type {
ProcessorConfig,
AsyncProcessorRuntime,
BackendConsumer,
BackendProducer,
LibrarianRequest,
LibrarianResponse,
CollectionManagementRequest,
CollectionManagementResponse,
DocumentMetadata,
ProcessingMetadata,
} from "@trustgraph/base";
import {
errorMessage,
makeAsyncProcessor,
makeProcessorProgram,
type ProcessorConfig,
type AsyncProcessorRuntime,
type BackendConsumer,
type BackendProducer,
topics,
type LibrarianRequest,
type LibrarianResponse,
type CollectionManagementRequest,
type CollectionManagementResponse,
DocumentMetadata as DocumentMetadataSchema,
type DocumentMetadata,
ProcessingMetadata as ProcessingMetadataSchema,
type ProcessingMetadata,
Triple as TripleSchema,
processorLifecycleError,
} from "@trustgraph/base";
import type { Message } from "@trustgraph/base";
import { NodeRuntime } from "@effect/platform-node";
import { Clock, Config, DateTime, Duration, Effect, Match, Option, Random, SynchronizedRef } from "effect";
import * as A from "effect/Array";
import * as MutableHashMap from "effect/MutableHashMap";
import * as S from "effect/Schema";
import { makeCollectionManager, type CollectionManager } from "./collection-manager.js";
import type { CollectionManager } from "./collection-manager.js";
import { makeCollectionManager, } from "./collection-manager.js";
import {
ensureDirectoryEffect,
joinPath,
@ -127,13 +131,10 @@ const decodePersistedLibrarianState = (
);
const randomUuid: Effect.Effect<string> = Effect.gen(function* () {
const bytes: number[] = [];
for (let index = 0; index < 16; index += 1) {
bytes.push(yield* Random.nextIntBetween(0, 255));
}
bytes[6] = (bytes[6]! & 0x0f) | 0x40;
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
const raw = yield* Effect.forEach(A.range(0, 15), () => Random.nextIntBetween(0, 255));
const bytes = A.map(raw, (byte, index) =>
index === 6 ? (byte & 0x0f) | 0x40 : index === 8 ? (byte & 0x3f) | 0x80 : byte,
);
const hex = bytes.map((byte) => byte.toString(16).padStart(2, "0"));
return [
@ -760,25 +761,24 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS
handleLibrarianOperation: function(this: LibrarianService, request: LibrarianRequest): Effect.Effect<LibrarianResponse, LibrarianServiceError> {
const service = this;
return Match.value(request.operation).pipe(
Match.when("add-document", () => service.addDocument(request)),
Match.when("remove-document", () => service.removeDocument(request)),
Match.when("update-document", () => service.updateDocument(request)),
Match.when("list-documents", () => service.listDocuments(request)),
Match.when("add-document", () => this.addDocument(request)),
Match.when("remove-document", () => this.removeDocument(request)),
Match.when("update-document", () => this.updateDocument(request)),
Match.when("list-documents", () => this.listDocuments(request)),
Match.when("get-document-metadata", () => getDocumentMetadataEffect(request)),
Match.when("get-document-content", () => service.getDocumentContent(request)),
Match.when("add-child-document", () => service.addChildDocument(request)),
Match.when("get-document-content", () => this.getDocumentContent(request)),
Match.when("add-child-document", () => this.addChildDocument(request)),
Match.when("list-children", () => listChildrenEffect(request)),
Match.when("add-processing", () => service.addProcessing(request)),
Match.when("remove-processing", () => service.removeProcessing(request)),
Match.when("list-processing", () => service.listProcessing(request)),
Match.when("begin-upload", () => service.beginUpload(request)),
Match.when("add-processing", () => this.addProcessing(request)),
Match.when("remove-processing", () => this.removeProcessing(request)),
Match.when("list-processing", () => this.listProcessing(request)),
Match.when("begin-upload", () => this.beginUpload(request)),
Match.when("upload-chunk", () => uploadChunkEffect(request)),
Match.when("complete-upload", () => service.completeUpload(request)),
Match.when("complete-upload", () => this.completeUpload(request)),
Match.when("get-upload-status", () => getUploadStatusEffect(request)),
Match.when("abort-upload", () => abortUploadEffect(request)),
Match.when("list-uploads", () => service.listUploads(request)),
Match.when("list-uploads", () => this.listUploads(request)),
Match.when("stream-document", () =>
Effect.fail(
librarianServiceError("stream-document", "stream-document must be handled as a streaming operation"),
@ -1427,7 +1427,7 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS
// ---------- Persistence ----------
persist: Effect.gen(function* () {
const current = service!;
const current = yield* getService;
const serviceState = yield* SynchronizedRef.get(current.state);
const data = {
documents: Object.fromEntries(serviceState.documents),
@ -1448,7 +1448,7 @@ export function makeLibrarianService(config: LibrarianServiceConfig): LibrarianS
loadFromDisk: Effect.gen(function* () {
const current = service!;
const current = yield* getService;
const parsed = yield* Effect.gen(function* () {
const raw = yield* readTextFileEffect(current.persistPath).pipe(
Effect.mapError((cause) => librarianServiceError("persist-read", cause)),

16
ts/packages/flow/src/model/text-completion/azure-openai.ts
View file

@ -10,16 +10,22 @@
import { AzureOpenAI } from "openai";
import { NodeRuntime } from "@effect/platform-node";
import type {
Llm,
LlmProvider,
ProcessorConfig,
LlmResult,
} from "@trustgraph/base";
import {
makeLlmService,
makeFlowProcessorProgram,
makeLlmSpecs,
type Llm,
type LlmProvider,
type ProcessorConfig,
type LlmResult,
} from "@trustgraph/base";
import { Effect, Stream } from "effect";
import type {
TextCompletionConfigError,
TextCompletionRuntimeError,
} from "./common.ts";
import {
llmStreamPart,
makeTextCompletionLayer,
@ -27,8 +33,6 @@ import {
providerStatusError,
requiredString,
streamTextCompletionChunks,
type TextCompletionConfigError,
type TextCompletionRuntimeError,
} from "./common.ts";
export type AzureOpenAIProcessorConfig = ProcessorConfig & {

14
ts/packages/flow/src/model/text-completion/claude.ts
View file

@ -6,23 +6,27 @@
import { AnthropicClient, AnthropicLanguageModel } from "@effect/ai-anthropic";
import { NodeRuntime } from "@effect/platform-node";
import type {
Llm,
LlmProvider,
ProcessorConfig,
} from "@trustgraph/base";
import {
makeLlmService,
makeFlowProcessorProgram,
makeLlmSpecs,
type Llm,
type LlmProvider,
type ProcessorConfig,
} from "@trustgraph/base";
import { Effect, Layer, Redacted } from "effect";
import { FetchHttpClient } from "effect/unstable/http";
import type {
TextCompletionConfigError,
TextCompletionRuntimeError,
} from "./common.ts";
import {
makeLanguageModelProvider,
makeTextCompletionLayer,
optionalStringConfig,
requiredString,
type TextCompletionConfigError,
type TextCompletionRuntimeError,
} from "./common.ts";
export type ClaudeProcessorConfig = ProcessorConfig & {

14
ts/packages/flow/src/model/text-completion/common.ts
View file

@ -1,18 +1,22 @@
import type {
LlmChunk,
LlmResult,
LlmProvider,
} from "@trustgraph/base";
import {
Llm,
TooManyRequestsError,
errorMessage,
makeLlmServiceShape,
type LlmChunk,
type LlmResult,
type LlmProvider,
} from "@trustgraph/base";
import { Config, Context, Effect, Layer, Match, Ref, Result, Stream } from "effect";
import type { Context, } from "effect";
import { Config, Effect, Layer, Match, Ref, Result, Stream } from "effect";
import * as O from "effect/Option";
import * as Predicate from "effect/Predicate";
import * as S from "effect/Schema";
import type * as Scope from "effect/Scope";
import { AiError, LanguageModel, Prompt, Response } from "effect/unstable/ai";
import type { LanguageModel, Prompt, Response } from "effect/unstable/ai";
import { AiError, } from "effect/unstable/ai";
export class TextCompletionConfigError extends S.TaggedErrorClass<TextCompletionConfigError>()(
"TextCompletionConfigError",

16
ts/packages/flow/src/model/text-completion/mistral.ts
View file

@ -8,16 +8,22 @@
import { Mistral } from "@mistralai/mistralai";
import { NodeRuntime } from "@effect/platform-node";
import type {
Llm,
LlmProvider,
ProcessorConfig,
LlmResult,
} from "@trustgraph/base";
import {
makeLlmService,
makeFlowProcessorProgram,
makeLlmSpecs,
type Llm,
type LlmProvider,
type ProcessorConfig,
type LlmResult,
} from "@trustgraph/base";
import { Effect, Stream } from "effect";
import type {
TextCompletionConfigError,
TextCompletionRuntimeError,
} from "./common.ts";
import {
llmStreamPart,
makeTextCompletionLayer,
@ -26,8 +32,6 @@ import {
requiredString,
streamTextCompletionChunks,
textFromContent,
type TextCompletionConfigError,
type TextCompletionRuntimeError,
} from "./common.ts";
export type MistralProcessorConfig = ProcessorConfig & {

20
ts/packages/flow/src/model/text-completion/ollama.ts
View file

@ -8,24 +8,28 @@
import { Ollama } from "ollama";
import { NodeRuntime } from "@effect/platform-node";
import type {
Llm,
LlmProvider,
ProcessorConfig,
LlmResult,
} from "@trustgraph/base";
import {
makeLlmService,
makeFlowProcessorProgram,
makeLlmSpecs,
type Llm,
type LlmProvider,
type ProcessorConfig,
type LlmResult,
} from "@trustgraph/base";
import { Effect, Stream } from "effect";
import type {
TextCompletionConfigError,
TextCompletionRuntimeError,
} from "./common.ts";
import {
llmStreamPart,
makeTextCompletionLayer,
optionalStringConfig,
providerRuntimeError,
streamTextCompletionChunks,
type TextCompletionConfigError,
type TextCompletionRuntimeError,
} from "./common.ts";
export type OllamaProcessorConfig = ProcessorConfig & {
@ -68,7 +72,7 @@ const makeOllamaProviderFromClient = (
_temperature?: number,
) => {
const modelName = model ?? defaultModel;
const fullPrompt = system + "\n\n" + prompt;
const fullPrompt = `${system}\n\n${prompt}`;
return Effect.tryPromise({
try: () =>
@ -95,7 +99,7 @@ const makeOllamaProviderFromClient = (
_temperature?: number,
) => {
const modelName = model ?? defaultModel;
const fullPrompt = system + "\n\n" + prompt;
const fullPrompt = `${system}\n\n${prompt}`;
return Stream.fromEffect(
Effect.tryPromise({

16
ts/packages/flow/src/model/text-completion/openai-compatible.ts
View file

@ -11,16 +11,22 @@
import OpenAI from "openai";
import { NodeRuntime } from "@effect/platform-node";
import type {
Llm,
LlmProvider,
ProcessorConfig,
LlmResult,
} from "@trustgraph/base";
import {
makeLlmService,
makeFlowProcessorProgram,
makeLlmSpecs,
type Llm,
type LlmProvider,
type ProcessorConfig,
type LlmResult,
} from "@trustgraph/base";
import { Effect, Stream } from "effect";
import type {
TextCompletionConfigError,
TextCompletionRuntimeError,
} from "./common.ts";
import {
llmStreamPart,
makeTextCompletionLayer,
@ -28,8 +34,6 @@ import {
providerStatusError,
requiredString,
streamTextCompletionChunks,
type TextCompletionConfigError,
type TextCompletionRuntimeError,
} from "./common.ts";
export type OpenAICompatibleProcessorConfig = ProcessorConfig & {

16
ts/packages/flow/src/model/text-completion/openai.ts
View file

@ -6,16 +6,22 @@
import OpenAI from "openai";
import { NodeRuntime } from "@effect/platform-node";
import type {
Llm,
LlmProvider,
ProcessorConfig,
LlmResult,
} from "@trustgraph/base";
import {
makeLlmService,
makeFlowProcessorProgram,
makeLlmSpecs,
type Llm,
type LlmProvider,
type ProcessorConfig,
type LlmResult,
} from "@trustgraph/base";
import { Effect, Stream } from "effect";
import type {
TextCompletionConfigError,
TextCompletionRuntimeError,
} from "./common.ts";
import {
llmStreamPart,
makeTextCompletionLayer,
@ -23,8 +29,6 @@ import {
providerStatusError,
requiredString,
streamTextCompletionChunks,
type TextCompletionConfigError,
type TextCompletionRuntimeError,
} from "./common.ts";
export type OpenAIProcessorConfig = ProcessorConfig & {

20
ts/packages/flow/src/prompt/template.ts
View file

@ -24,19 +24,21 @@
* Python reference: trustgraph-flow/trustgraph/prompt/template/service.py
*/
import type {
ProcessorConfig,
EffectConfigHandler,
FlowContext,
FlowProcessorRuntime,
FlowResourceNotFoundError,
MessagingDeliveryError,
PromptRequest,
PromptResponse,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
makeProducerSpec,
type ProcessorConfig,
type EffectConfigHandler,
type FlowContext,
type FlowProcessorRuntime,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type PromptRequest,
type PromptResponse,
type Spec,
} from "@trustgraph/base";
import { NodeRuntime } from "@effect/platform-node";
import { makeFlowProcessorProgram } from "@trustgraph/base";

3
ts/packages/flow/src/qdrant/client.ts
View file

@ -1,4 +1,5 @@
import { QdrantClient, type QdrantClientParams } from "@qdrant/js-client-rest";
import type { QdrantClientParams } from "@qdrant/js-client-rest";
import { QdrantClient, } from "@qdrant/js-client-rest";
import { errorMessage } from "@trustgraph/base";
import { Effect } from "effect";
import * as S from "effect/Schema";

26
ts/packages/flow/src/query/embeddings/qdrant-doc-service.ts
View file

@ -7,30 +7,34 @@
* Python reference: trustgraph-flow/trustgraph/query/doc_embeddings/qdrant/service.py
*/
import type {
ProcessorConfig,
FlowProcessorRuntime,
FlowProcessorStartEffect,
FlowContext,
FlowResourceNotFoundError,
MessagingDeliveryError,
DocumentEmbeddingsRequest,
DocumentEmbeddingsResponse,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
makeProducerSpec,
processorLifecycleError,
type ProcessorConfig,
type FlowProcessorRuntime,
type FlowProcessorStartEffect,
type FlowContext,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type DocumentEmbeddingsRequest,
type DocumentEmbeddingsResponse,
type Spec,
} from "@trustgraph/base";
import { NodeRuntime } from "@effect/platform-node";
import { makeFlowProcessorProgram } from "@trustgraph/base";
import { Effect } from "effect";
import type {
QdrantDocQueryConfig,
QdrantDocEmbeddingsQueryError,
} from "./qdrant-doc.js";
import {
QdrantDocEmbeddingsQueryLive,
QdrantDocEmbeddingsQueryService,
makeQdrantDocEmbeddingsQueryServiceEffect,
type QdrantDocQueryConfig,
type QdrantDocEmbeddingsQueryError,
} from "./qdrant-doc.js";
const DocumentEmbeddingsResponseProducer = makeProducerSpec<DocumentEmbeddingsResponse>("document-embeddings-response");

3
ts/packages/flow/src/query/embeddings/qdrant-doc.ts
View file

@ -11,7 +11,8 @@ import { errorMessage } from "@trustgraph/base";
import { Config, Context, Effect, Layer } from "effect";
import * as O from "effect/Option";
import * as S from "effect/Schema";
import { makeQdrantClient, type QdrantClientFactory, type QdrantClientLike } from "../../qdrant/client.js";
import type { QdrantClientFactory, QdrantClientLike } from "../../qdrant/client.js";
import { makeQdrantClient, } from "../../qdrant/client.js";
export interface QdrantDocQueryConfig {
url?: string;

26
ts/packages/flow/src/query/embeddings/qdrant-graph-service.ts
View file

@ -7,30 +7,34 @@
* Python reference: trustgraph-flow/trustgraph/query/graph_embeddings/qdrant/service.py
*/
import type {
ProcessorConfig,
FlowProcessorRuntime,
FlowProcessorStartEffect,
FlowContext,
FlowResourceNotFoundError,
MessagingDeliveryError,
GraphEmbeddingsRequest,
GraphEmbeddingsResponse,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
makeProducerSpec,
processorLifecycleError,
type ProcessorConfig,
type FlowProcessorRuntime,
type FlowProcessorStartEffect,
type FlowContext,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type GraphEmbeddingsRequest,
type GraphEmbeddingsResponse,
type Spec,
} from "@trustgraph/base";
import { NodeRuntime } from "@effect/platform-node";
import { makeFlowProcessorProgram } from "@trustgraph/base";
import { Effect } from "effect";
import type {
QdrantGraphQueryConfig,
QdrantGraphEmbeddingsQueryError,
} from "./qdrant-graph.js";
import {
QdrantGraphEmbeddingsQueryLive,
QdrantGraphEmbeddingsQueryService,
makeQdrantGraphEmbeddingsQueryServiceEffect,
type QdrantGraphQueryConfig,
type QdrantGraphEmbeddingsQueryError,
} from "./qdrant-graph.js";
const GraphEmbeddingsResponseProducer = makeProducerSpec<GraphEmbeddingsResponse>("graph-embeddings-response");

6
ts/packages/flow/src/query/embeddings/qdrant-graph.ts
View file

@ -10,11 +10,13 @@
* Python reference: trustgraph-flow/trustgraph/query/graph_embeddings/qdrant/service.py
*/
import { errorMessage, type Term } from "@trustgraph/base";
import type { Term } from "@trustgraph/base";
import { errorMessage, } from "@trustgraph/base";
import { Config, Context, Effect, Layer } from "effect";
import * as O from "effect/Option";
import * as S from "effect/Schema";
import { makeQdrantClient, type QdrantClientFactory, type QdrantClientLike } from "../../qdrant/client.js";
import type { QdrantClientFactory, QdrantClientLike } from "../../qdrant/client.js";
import { makeQdrantClient, } from "../../qdrant/client.js";
export interface QdrantGraphQueryConfig {
url?: string;

26
ts/packages/flow/src/query/triples/falkordb-service.ts
View file

@ -7,30 +7,34 @@
* Python reference: trustgraph-flow/trustgraph/query/triples/falkordb/service.py
*/
import type {
ProcessorConfig,
FlowProcessorRuntime,
FlowProcessorStartEffect,
FlowContext,
FlowResourceNotFoundError,
MessagingDeliveryError,
TriplesQueryRequest,
TriplesQueryResponse,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
makeProducerSpec,
processorLifecycleError,
type ProcessorConfig,
type FlowProcessorRuntime,
type FlowProcessorStartEffect,
type FlowContext,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type TriplesQueryRequest,
type TriplesQueryResponse,
type Spec,
} from "@trustgraph/base";
import { NodeRuntime } from "@effect/platform-node";
import { makeFlowProcessorProgram } from "@trustgraph/base";
import { Effect } from "effect";
import type {
FalkorDBQueryConfig,
FalkorDBTriplesQueryError,
} from "./falkordb.js";
import {
FalkorDBTriplesQueryLive,
FalkorDBTriplesQueryService,
makeFalkorDBTriplesQueryServiceScoped,
type FalkorDBQueryConfig,
type FalkorDBTriplesQueryError,
} from "./falkordb.js";
const TriplesResponseProducer = makeProducerSpec<TriplesQueryResponse>("triples-response");

3
ts/packages/flow/src/query/triples/falkordb.ts
View file

@ -7,7 +7,8 @@
*/
import { createClient, Graph } from "falkordb";
import { errorMessage, type Term, type Triple } from "@trustgraph/base";
import type { Term, Triple } from "@trustgraph/base";
import { errorMessage, } from "@trustgraph/base";
import { Config, Context, Effect, Layer, Match } from "effect";
import * as Predicate from "effect/Predicate";
import * as S from "effect/Schema";

40
ts/packages/flow/src/retrieval/document-rag-service.ts
View file

@ -8,36 +8,40 @@
*/
import {NodeRuntime} from "@effect/platform-node";
import type {
DocumentEmbeddingsRequest,
DocumentEmbeddingsResponse,
DocumentRagRequest,
DocumentRagResponse,
EmbeddingsRequest,
EmbeddingsResponse,
FlowContext,
FlowProcessorRuntime,
FlowResourceNotFoundError,
MessagingDeliveryError,
ProcessorConfig,
PromptRequest,
PromptResponse,
Spec,
TextCompletionRequest,
TextCompletionResponse,
} from "@trustgraph/base";
import {
makeConsumerSpec,
makeFlowProcessor,
makeProducerSpec,
makeRequestResponseSpec,
makeFlowProcessorProgram,
type DocumentEmbeddingsRequest,
type DocumentEmbeddingsResponse,
type DocumentRagRequest,
type DocumentRagResponse,
type EmbeddingsRequest,
type EmbeddingsResponse,
type FlowContext,
type FlowProcessorRuntime,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type ProcessorConfig,
type PromptRequest,
type PromptResponse,
type Spec,
type TextCompletionRequest,
type TextCompletionResponse,
} from "@trustgraph/base";
import {Effect} from "effect";
import type {
DocumentRagEngineError,
DocumentRagClients,
} from "./document-rag.js";
import {
DocumentRagEngine,
DocumentRagEngineError,
DocumentRagLive,
makeDocumentRagEngine,
type DocumentRagClients,
} from "./document-rag.js";
const DocumentRagResponseProducer = makeProducerSpec<DocumentRagResponse>("document-rag-response");

46
ts/packages/flow/src/retrieval/graph-rag-service.ts
View file

@ -8,39 +8,43 @@
*/
import {NodeRuntime} from "@effect/platform-node";
import type {
FlowContext,
FlowProcessorRuntime,
FlowResourceNotFoundError,
GraphEmbeddingsRequest,
GraphEmbeddingsResponse,
GraphRagRequest,
GraphRagResponse,
EmbeddingsRequest,
EmbeddingsResponse,
MessagingDeliveryError,
ProcessorConfig,
PromptRequest,
PromptResponse,
Spec,
TextCompletionRequest,
TextCompletionResponse,
TriplesQueryRequest,
TriplesQueryResponse,
} from "@trustgraph/base";
import {
makeConsumerSpec,
makeFlowProcessor,
makeProducerSpec,
makeRequestResponseSpec,
makeFlowProcessorProgram,
type FlowContext,
type FlowProcessorRuntime,
type FlowResourceNotFoundError,
type GraphEmbeddingsRequest,
type GraphEmbeddingsResponse,
type GraphRagRequest,
type GraphRagResponse,
type EmbeddingsRequest,
type EmbeddingsResponse,
type MessagingDeliveryError,
type ProcessorConfig,
type PromptRequest,
type PromptResponse,
type Spec,
type TextCompletionRequest,
type TextCompletionResponse,
type TriplesQueryRequest,
type TriplesQueryResponse,
} from "@trustgraph/base";
import {Effect} from "effect";
import type {
GraphRagEngineError,
GraphRagClients,
GraphRagConfig,
} from "./graph-rag.js";
import {
GraphRagEngine,
GraphRagEngineError,
GraphRagLive,
makeGraphRagEngine,
type GraphRagClients,
type GraphRagConfig,
} from "./graph-rag.js";
const GraphRagResponseProducer = makeProducerSpec<GraphRagResponse>("graph-rag-response");

32
ts/packages/flow/src/storage/embeddings/graph-embeddings-service.ts
View file

@ -9,33 +9,37 @@
* Python reference: trustgraph-flow/trustgraph/storage/graph_embeddings/qdrant/service.py
*/
import type {
ProcessorConfig,
FlowProcessorRuntime,
FlowProcessorStartEffect,
FlowContext,
FlowResourceNotFoundError,
MessagingDeliveryError,
MessagingLifecycleError,
MessagingTimeoutError,
EntityContexts,
EmbeddingsRequest,
EmbeddingsResponse,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
makeRequestResponseSpec,
processorLifecycleError,
type ProcessorConfig,
type FlowProcessorRuntime,
type FlowProcessorStartEffect,
type FlowContext,
type FlowResourceNotFoundError,
type MessagingDeliveryError,
type MessagingLifecycleError,
type MessagingTimeoutError,
type EntityContexts,
type EmbeddingsRequest,
type EmbeddingsResponse,
type Spec,
} from "@trustgraph/base";
import { NodeRuntime } from "@effect/platform-node";
import { makeFlowProcessorProgram } from "@trustgraph/base";
import { Effect } from "effect";
import type {
QdrantGraphEmbeddingsConfig,
QdrantGraphEmbeddingsStoreError,
} from "./qdrant-graph.js";
import {
QdrantGraphEmbeddingsStoreLive,
QdrantGraphEmbeddingsStoreService,
makeQdrantGraphEmbeddingsStoreServiceEffect,
type QdrantGraphEmbeddingsConfig,
type QdrantGraphEmbeddingsStoreError,
} from "./qdrant-graph.js";
type GraphEmbeddingsStoreRequirements = QdrantGraphEmbeddingsStoreService;

3
ts/packages/flow/src/storage/embeddings/qdrant-doc.ts
View file

@ -13,7 +13,8 @@ import { Config, Effect, Random } from "effect";
import * as MutableHashSet from "effect/MutableHashSet";
import * as O from "effect/Option";
import * as S from "effect/Schema";
import { makeQdrantClient, type QdrantClientFactory, type QdrantClientLike } from "../../qdrant/client.js";
import type { QdrantClientFactory, QdrantClientLike } from "../../qdrant/client.js";
import { makeQdrantClient, } from "../../qdrant/client.js";
export interface QdrantDocEmbeddingsConfig {
url?: string;

6
ts/packages/flow/src/storage/embeddings/qdrant-graph.ts
View file

@ -8,12 +8,14 @@
* Python reference: trustgraph-flow/trustgraph/storage/graph_embeddings/qdrant/write.py
*/
import { errorMessage, type Term } from "@trustgraph/base";
import type { Term } from "@trustgraph/base";
import { errorMessage, } from "@trustgraph/base";
import { Config, Context, Effect, Layer, Match, Random } from "effect";
import * as MutableHashSet from "effect/MutableHashSet";
import * as O from "effect/Option";
import * as S from "effect/Schema";
import { makeQdrantClient, type QdrantClientFactory, type QdrantClientLike } from "../../qdrant/client.js";
import type { QdrantClientFactory, QdrantClientLike } from "../../qdrant/client.js";
import { makeQdrantClient, } from "../../qdrant/client.js";
export interface QdrantGraphEmbeddingsConfig {
url?: string;

20
ts/packages/flow/src/storage/triples/falkordb-service.ts
View file

@ -8,26 +8,30 @@
* Python reference: trustgraph-flow/trustgraph/storage/triples/falkordb/service.py
*/
import type {
ProcessorConfig,
FlowProcessorRuntime,
FlowProcessorStartEffect,
FlowContext,
Triples,
Spec,
} from "@trustgraph/base";
import {
makeFlowProcessor,
makeConsumerSpec,
processorLifecycleError,
type ProcessorConfig,
type FlowProcessorRuntime,
type FlowProcessorStartEffect,
type FlowContext,
type Triples,
type Spec,
} from "@trustgraph/base";
import { NodeRuntime } from "@effect/platform-node";
import { makeFlowProcessorProgram } from "@trustgraph/base";
import { Effect } from "effect";
import type {
FalkorDBConfig,
FalkorDBTriplesStoreError,
} from "./falkordb.js";
import {
FalkorDBTriplesStoreLive,
FalkorDBTriplesStoreService,
makeFalkorDBTriplesStoreServiceScoped,
type FalkorDBConfig,
type FalkorDBTriplesStoreError,
} from "./falkordb.js";
const onStoreTriplesMessage = Effect.fn("TriplesStoreService.onMessage")(function* (

3
ts/packages/flow/src/storage/triples/falkordb.ts
View file

@ -8,7 +8,8 @@
*/
import { createClient, Graph } from "falkordb";
import { errorMessage, type Term, type Triple } from "@trustgraph/base";
import type { Term, Triple } from "@trustgraph/base";
import { errorMessage, } from "@trustgraph/base";
import { Config, Context, Effect, Layer, Match } from "effect";
import * as S from "effect/Schema";

3
ts/packages/mcp/package.json
View file

@ -8,7 +8,8 @@
"build": "bunx --bun tsc",
"dev": "tsc --watch",
"clean": "rm -rf dist",
"test": "bunx --bun vitest run --passWithNoTests --exclude=dist/**"
"test": "bunx --bun vitest run --passWithNoTests --exclude=dist/**",
"lint": "bunx --bun biome check src"
},
"dependencies": {
"@trustgraph/base": "workspace:*",

5
ts/packages/mcp/src/__tests__/server-effect.test.ts
View file

@ -1,5 +1,6 @@
import { describe, expect, it } from "@effect/vitest";
import { DispatchStreamChunk, type BaseApi, type TrustGraphGatewayClient } from "@trustgraph/client";
import type { BaseApi, TrustGraphGatewayClient } from "@trustgraph/client";
import { DispatchStreamChunk, } from "@trustgraph/client";
import { Effect, Layer, Stream } from "effect";
import * as S from "effect/Schema";
import { McpServer } from "effect/unstable/ai";
@ -226,7 +227,7 @@ const makeNativeTestClientEffect = Effect.fn("makeNativeTestClient")(function*(
const textContent = (result: McpSchema.CallToolResult): string => {
const [content] = result.content;
expect(content?.type).toBe("text");
return "text" in content! ? content.text : "";
return content !== undefined && "text" in content ? content.text : "";
};
describe("Effect MCP server", () => {

8
ts/packages/mcp/src/server-effect.ts
View file

@ -1,11 +1,13 @@
import {BunHttpServer, BunRuntime} from "@effect/platform-bun";
import {NodeRuntime, NodeStdio} from "@effect/platform-node";
import type {
BaseApi,
Term as ClientTerm,
TrustGraphGatewayClient,
} from "@trustgraph/client";
import {
createTrustGraphSocket,
makeTrustGraphGatewayClientScoped,
type BaseApi,
type Term as ClientTerm,
type TrustGraphGatewayClient,
} from "@trustgraph/client";
import {Config, Context, Effect, Layer} from "effect";
import * as O from "effect/Option";

3
ts/packages/workbench/package.json
View file

@ -7,7 +7,8 @@
"dev": "vite",
"build": "tsc && vite build",
"preview": "vite preview",
"qa:browser": "playwright test"
"qa:browser": "playwright test",
"lint": "bunx --bun biome check src"
},
"dependencies": {
"@effect/atom-react": "4.0.0-beta.78",

49
ts/packages/workbench/src/atoms/workbench.ts
View file

@ -1,25 +1,29 @@
import { Clipboard as BrowserClipboard } from "@effect/platform-browser";
import * as BrowserHttpClient from "@effect/platform-browser/BrowserHttpClient";
import * as BrowserKeyValueStore from "@effect/platform-browser/BrowserKeyValueStore";
import type {
GraphRagOptions,
BaseApi,
BeginUploadResponse,
ChunkedUploadDocumentMetadata,
CompleteUploadResponse,
ConnectionState,
DocumentMetadata,
ExplainEvent,
StreamingMetadata,
Term,
Triple,
UploadChunkResponse,
} from "@trustgraph/client";
import {
DispatchPayload,
GatewayWorkbenchHttpApi,
type GraphRagOptions,
makeBaseApi,
TrustGraphRpcs,
type BaseApi,
type BeginUploadResponse,
type ChunkedUploadDocumentMetadata,
type CompleteUploadResponse,
type ConnectionState,
type DocumentMetadata,
type ExplainEvent,
type StreamingMetadata,
type Term,
type Triple,
type UploadChunkResponse,
} from "@trustgraph/client";
import { Cause, Clock, Context, Effect, Layer, Match, Metric, Option, Random, Schema as S, Scope, Stream } from "effect";
import type { Scope, } from "effect";
import { Cause, Clock, Context, Effect, Layer, Match, Metric, Option, Random, Schema as S, Stream } from "effect";
import * as A from "effect/Array";
import * as MutableHashMap from "effect/MutableHashMap";
import * as Predicate from "effect/Predicate";
import { HttpClient, HttpClientRequest } from "effect/unstable/http";
@ -28,7 +32,7 @@ import * as RpcClient from "effect/unstable/rpc/RpcClient";
import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization";
import * as AsyncResult from "effect/unstable/reactivity/AsyncResult";
import * as Atom from "effect/unstable/reactivity/Atom";
import * as AtomRegistry from "effect/unstable/reactivity/AtomRegistry";
import type * as AtomRegistry from "effect/unstable/reactivity/AtomRegistry";
import * as AtomHttpApi from "effect/unstable/reactivity/AtomHttpApi";
import * as AtomRpc from "effect/unstable/reactivity/AtomRpc";
import * as Reactivity from "effect/unstable/reactivity/Reactivity";
@ -1490,14 +1494,15 @@ function updateConversation(get: Atom.FnContext, f: (current: ConversationState)
}
function updateLastMessage(get: Atom.FnContext, updater: (prev: ChatMessage) => ChatMessage): void {
updateConversation(get, (current) => {
if (current.messages.length === 0) return current;
const last = current.messages[current.messages.length - 1]!;
return {
...current,
messages: [...current.messages.slice(0, -1), updater(last)],
};
});
updateConversation(get, (current) =>
A.matchRight(current.messages, {
onEmpty: () => current,
onNonEmpty: (init, last) => ({
...current,
messages: [...init, updater(last)],
}),
}),
);
}
function refreshConfigAtoms(get: Atom.FnContext): void {

6
ts/packages/workbench/src/components/chat/explain-graph.tsx
View file

@ -9,11 +9,13 @@ import {
resultError,
resultLoading,
} from "@/atoms/workbench";
import type {
GraphNode,
GraphLink,
} from "@/lib/graph-utils";
import {
triplesToGraph,
localName,
type GraphNode,
type GraphLink,
directedGraphLinkProps,
DEFAULT_GRAPH_NODE_COLOR,
} from "@/lib/graph-utils";

4
ts/packages/workbench/src/components/error-boundary.tsx
View file

@ -1,7 +1,9 @@
import type { ReactNode } from "react";
import type {
FallbackProps,
} from "react-error-boundary";
import {
ErrorBoundary as ReactErrorBoundary,
type FallbackProps,
} from "react-error-boundary";
import { AlertTriangle, RefreshCw } from "lucide-react";
import { Effect } from "effect";

3
ts/packages/workbench/src/components/notification-toasts.tsx
View file

@ -1,7 +1,8 @@
import { useAtomSet, useAtomValue } from "@effect/atom-react";
import { X } from "lucide-react";
import { cn } from "@/lib/utils";
import { notificationsAtom, removeNotificationAtom, type Notification } from "@/atoms/workbench";
import type { Notification } from "@/atoms/workbench";
import { notificationsAtom, removeNotificationAtom, } from "@/atoms/workbench";
const typeStyles: Record<Notification["type"], string> = {
success: "border-success/40 bg-success/10 text-success",

4
ts/packages/workbench/src/index.css
View file

@ -134,8 +134,8 @@
.animate-\[glow-drift-2_15s_ease-in-out_infinite\],
.animate-\[glow-drift-3_25s_ease-in-out_infinite\],
.animate-\[glow-fade-in_1\.2s_ease-out_forwards\] {
animation: none !important;
opacity: 0.7 !important;
animation: none;
opacity: 0.7;
}
}

28
ts/packages/workbench/src/lib/graph-utils.ts
View file

@ -123,16 +123,18 @@ export function triplesToGraph(triples: Triple[]): {
const nodeMap = new Map<string, GraphNode>();
const links: GraphLink[] = [];
const ensureNode = (uri: string): void => {
if (!nodeMap.has(uri)) {
const type = typeMap.get(uri);
nodeMap.set(uri, {
id: uri,
label: labelMap.get(uri) ?? localName(uri),
color: hashColor(type !== undefined ? localName(type) : uri),
degree: 0,
});
}
const ensureNode = (uri: string): GraphNode => {
const existing = nodeMap.get(uri);
if (existing !== undefined) return existing;
const type = typeMap.get(uri);
const node: GraphNode = {
id: uri,
label: labelMap.get(uri) ?? localName(uri),
color: hashColor(type !== undefined ? localName(type) : uri),
degree: 0,
};
nodeMap.set(uri, node);
return node;
};
for (const t of triples) {
@ -151,10 +153,8 @@ export function triplesToGraph(triples: Triple[]): {
const oIsEntity = isIri(t.o) || t.o.t === "l";
if (!sIsEntity || !oIsEntity) continue;
ensureNode(sVal);
ensureNode(oVal);
nodeMap.get(sVal)!.degree++;
nodeMap.get(oVal)!.degree++;
ensureNode(sVal).degree++;
ensureNode(oVal).degree++;
links.push({
source: sVal,

3
ts/packages/workbench/src/lib/utils.ts
View file

@ -1,4 +1,5 @@
import { type ClassValue, clsx } from "clsx";
import type { ClassValue, } from "clsx";
import { clsx } from "clsx";
import { twMerge } from "tailwind-merge";
export function cn(...inputs: ClassValue[]) {

8
ts/packages/workbench/src/main.tsx
View file

@ -13,7 +13,13 @@ function AppRoot() {
return <App />;
}
ReactDOM.createRoot(document.getElementById("root")!).render(
const rootElement = document.getElementById("root");
if (rootElement === null) {
// Host boundary: the workbench cannot render without its mount point.
throw new Error("Workbench root element #root not found");
}
ReactDOM.createRoot(rootElement).render(
<React.StrictMode>
<RegistryProvider defaultIdleTTL={1_000} initialValues={getWorkbenchQaInitialValues()}>
<AppRoot />

4
ts/packages/workbench/src/pages/chat.tsx
View file

@ -15,6 +15,9 @@ import {
} from "lucide-react";
import Markdown from "react-markdown";
import { cn } from "@/lib/utils";
import type {
ChatMessage,
} from "@/atoms/workbench";
import {
agentPhaseExpandedAtom,
cancelChatAtom,
@ -27,7 +30,6 @@ import {
setConversationInputAtom,
settingsAtom,
submitMessageAtom,
type ChatMessage,
} from "@/atoms/workbench";
import { AutoTextarea } from "@/components/ui/textarea";
import { MessageActions } from "@/components/chat/message-actions";

6
ts/packages/workbench/src/pages/graph.tsx
View file

@ -19,14 +19,16 @@ import {
settingsAtom,
} from "@/atoms/workbench";
import type { Triple } from "@trustgraph/client";
import type {
GraphNode,
GraphLink,
} from "@/lib/graph-utils";
import {
localName,
triplesToGraph,
RDFS_LABEL,
RDF_TYPE,
termValue,
type GraphNode,
type GraphLink,
directedGraphLinkProps,
DEFAULT_GRAPH_NODE_COLOR,
} from "@/lib/graph-utils";

4
ts/packages/workbench/src/pages/library.tsx
View file

@ -16,6 +16,9 @@ import {
Hash,
} from "lucide-react";
import { cn } from "@/lib/utils";
import type {
UploadForm,
} from "@/atoms/workbench";
import {
documentMetadataAtom,
encodeJsonUnknownString,
@ -31,7 +34,6 @@ import {
submitUploadDocumentAtom,
uploadDialogOpenAtom,
uploadFormAtom,
type UploadForm,
} from "@/atoms/workbench";
import { Dialog } from "@/components/ui/dialog";
import { Badge } from "@/components/ui/badge";

6
ts/packages/workbench/src/pages/mcp-tools.tsx
View file

@ -15,6 +15,10 @@ import {
} from "lucide-react";
import { cn } from "@/lib/utils";
import { Dialog } from "@/components/ui/dialog";
import type {
McpServerEntry,
ToolEntry,
} from "@/atoms/workbench";
import {
deleteMcpServerAtom,
deleteMcpToolAtom,
@ -31,8 +35,6 @@ import {
resultLoading,
saveMcpServerAtom,
saveMcpToolAtom,
type McpServerEntry,
type ToolEntry,
} from "@/atoms/workbench";
const INPUT_CLASS =

4
ts/packages/workbench/src/pages/prompts.tsx
View file

@ -95,7 +95,7 @@ export default function PromptsPage() {
)}
{activeTab === "templates" && (
<div id="panel-templates" role="tabpanel" aria-labelledby="tab-templates" tabIndex={0} className="flex flex-1 flex-col gap-4 overflow-hidden">
<div id="panel-templates" role="tabpanel" aria-labelledby="tab-templates" className="flex flex-1 flex-col gap-4 overflow-hidden">
{loading && prompts.length === 0 && (
<div className="flex items-center justify-center py-12">
<Loader2 className="mr-2 h-5 w-5 animate-spin text-fg-subtle" />
@ -203,7 +203,7 @@ export default function PromptsPage() {
)}
{activeTab === "system" && (
<div id="panel-system" role="tabpanel" aria-labelledby="tab-system" tabIndex={0} className="flex flex-1 flex-col overflow-hidden rounded-lg border border-border">
<div id="panel-system" role="tabpanel" aria-labelledby="tab-system" className="flex flex-1 flex-col overflow-hidden rounded-lg border border-border">
<div className="border-b border-border bg-surface-100 px-4 py-3">
<h2 className="text-xs font-medium uppercase tracking-wider text-fg-muted">
System Prompt

Some files were not shown because too many files have changed in this diff Show more