mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-06-12 19:55:19 +02:00
Merge remote-tracking branch 'origin/hono' into hono-frontend
This commit is contained in:
tusharmagar
commit
869318e22b
22 changed files with 3976 additions and 151 deletions
|
|
@ -1,7 +1,8 @@
|
|||
#!/usr/bin/env node
|
||||
import yargs from 'yargs';
|
||||
import { hideBin } from 'yargs/helpers';
|
||||
import { app, modelConfig, updateState, importExample, listExamples, exportWorkflow } from '../dist/app.js';
|
||||
import { app, modelConfig, importExample, listExamples, exportWorkflow } from '../dist/app.js';
|
||||
import { runTui } from '../dist/tui/index.js';
|
||||
|
||||
yargs(hideBin(process.argv))
|
||||
|
||||
|
|
@ -36,6 +37,20 @@ yargs(hideBin(process.argv))
|
|||
});
|
||||
}
|
||||
)
|
||||
.command(
|
||||
"ui",
|
||||
"Launch the interactive Rowboat dashboard",
|
||||
(y) => y
|
||||
.option("server-url", {
|
||||
type: "string",
|
||||
description: "Rowboat server base URL",
|
||||
}),
|
||||
(argv) => {
|
||||
runTui({
|
||||
serverUrl: argv.serverUrl,
|
||||
});
|
||||
}
|
||||
)
|
||||
.command(
|
||||
"import",
|
||||
"Import an example workflow (--example) or custom workflow from file (--file)",
|
||||
|
|
|
|||
1699
apps/cli/package-lock.json
generated
1699
apps/cli/package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
|
@ -21,6 +21,7 @@
|
|||
"description": "",
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.9.1",
|
||||
"@types/react": "^18.3.12",
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"dependencies": {
|
||||
|
|
@ -29,17 +30,28 @@
|
|||
"@ai-sdk/openai": "^2.0.53",
|
||||
"@ai-sdk/openai-compatible": "^1.0.27",
|
||||
"@ai-sdk/provider": "^2.0.0",
|
||||
"@google-cloud/local-auth": "^3.0.1",
|
||||
"@hono/node-server": "^1.19.6",
|
||||
"@hono/standard-validator": "^0.1.5",
|
||||
"@modelcontextprotocol/sdk": "^1.20.2",
|
||||
"@openrouter/ai-sdk-provider": "^1.2.6",
|
||||
"ai": "^5.0.102",
|
||||
"awilix": "^12.0.5",
|
||||
"eventsource-parser": "^1.1.2",
|
||||
"google-auth-library": "^10.5.0",
|
||||
"googleapis": "^169.0.0",
|
||||
"hono": "^4.10.7",
|
||||
"hono-openapi": "^1.1.1",
|
||||
"ink": "^5.1.0",
|
||||
"ink-select-input": "^6.2.0",
|
||||
"ink-spinner": "^5.0.0",
|
||||
"ink-text-input": "^6.0.0",
|
||||
"json-schema-to-zod": "^2.6.1",
|
||||
"nanoid": "^5.1.6",
|
||||
"node-html-markdown": "^2.0.0",
|
||||
"ollama-ai-provider-v2": "^1.5.4",
|
||||
"react": "^18.3.1",
|
||||
"yaml": "^2.8.2",
|
||||
"yargs": "^18.0.0",
|
||||
"zod": "^4.1.12"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ export const Agent = z.object({
|
|||
name: z.string(),
|
||||
provider: z.string().optional(),
|
||||
model: z.string().optional(),
|
||||
description: z.string(),
|
||||
description: z.string().optional(),
|
||||
instructions: z.string(),
|
||||
tools: z.record(z.string(), ToolAttachment).optional(),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
import { WorkDir } from "../config/config.js";
|
||||
import fs from "fs/promises";
|
||||
import { glob } from "node:fs/promises";
|
||||
import path from "path";
|
||||
import z from "zod";
|
||||
import { Agent } from "./agents.js";
|
||||
import { parse, stringify } from "yaml";
|
||||
|
||||
const UpdateAgentSchema = Agent.omit({ name: true });
|
||||
|
||||
export interface IAgentsRepo {
|
||||
list(): Promise<z.infer<typeof Agent>[]>;
|
||||
|
|
@ -13,33 +17,76 @@ export interface IAgentsRepo {
|
|||
}
|
||||
|
||||
export class FSAgentsRepo implements IAgentsRepo {
|
||||
private readonly agentsDir = path.join(WorkDir, "agents");
|
||||
|
||||
async list(): Promise<z.infer<typeof Agent>[]> {
|
||||
const result: z.infer<typeof Agent>[] = [];
|
||||
// list all json files in workdir/agents/
|
||||
const agentsDir = path.join(WorkDir, "agents");
|
||||
const files = await fs.readdir(agentsDir);
|
||||
|
||||
for (const file of files) {
|
||||
const contents = await fs.readFile(path.join(agentsDir, file), "utf8");
|
||||
result.push(Agent.parse(JSON.parse(contents)));
|
||||
// list all md files in workdir/agents/
|
||||
const matches = await Array.fromAsync(glob("**/*.md", { cwd: this.agentsDir }));
|
||||
for (const file of matches) {
|
||||
try {
|
||||
const agent = await this.parseAgentMd(path.join(this.agentsDir, file));
|
||||
result.push(agent);
|
||||
} catch (error) {
|
||||
console.error(`Error parsing agent ${file}: ${error instanceof Error ? error.message : String(error)}`);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private async parseAgentMd(filePath: string): Promise<z.infer<typeof Agent>> {
|
||||
const raw = await fs.readFile(filePath, "utf8");
|
||||
|
||||
// strip the path prefix from the file name
|
||||
// and the .md extension
|
||||
const agentName = filePath
|
||||
.replace(this.agentsDir + "/", "")
|
||||
.replace(/\.md$/, "");
|
||||
let agent: z.infer<typeof Agent> = {
|
||||
name: agentName,
|
||||
instructions: raw,
|
||||
};
|
||||
let content = raw;
|
||||
|
||||
// check for frontmatter markers at start
|
||||
if (raw.startsWith("---")) {
|
||||
const end = raw.indexOf("\n---", 3);
|
||||
|
||||
if (end !== -1) {
|
||||
const fm = raw.slice(3, end).trim(); // YAML text
|
||||
content = raw.slice(end + 4).trim(); // body after frontmatter
|
||||
const yaml = parse(fm);
|
||||
const parsed = Agent
|
||||
.omit({ name: true, instructions: true })
|
||||
.parse(yaml);
|
||||
agent = {
|
||||
...agent,
|
||||
...parsed,
|
||||
instructions: content,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return agent;
|
||||
}
|
||||
|
||||
async fetch(id: string): Promise<z.infer<typeof Agent>> {
|
||||
const contents = await fs.readFile(path.join(WorkDir, "agents", `${id}.json`), "utf8");
|
||||
return Agent.parse(JSON.parse(contents));
|
||||
return this.parseAgentMd(path.join(this.agentsDir, `${id}.md`));
|
||||
}
|
||||
|
||||
async create(agent: z.infer<typeof Agent>): Promise<void> {
|
||||
await fs.writeFile(path.join(WorkDir, "agents", `${agent.name}.json`), JSON.stringify(agent, null, 2));
|
||||
await fs.writeFile(path.join(this.agentsDir, `${agent.name}.md`), agent.instructions);
|
||||
}
|
||||
|
||||
async update(id: string, agent: z.infer<typeof Agent>): Promise<void> {
|
||||
await fs.writeFile(path.join(WorkDir, "agents", `${id}.json`), JSON.stringify(agent, null, 2));
|
||||
|
||||
async update(id: string, agent: z.infer<typeof UpdateAgentSchema>): Promise<void> {
|
||||
const { instructions, ...rest } = agent;
|
||||
const contents = `---\n${stringify(rest)}\n---\n${instructions}`;
|
||||
await fs.writeFile(path.join(this.agentsDir, `${id}.md`), contents);
|
||||
}
|
||||
|
||||
async delete(id: string): Promise<void> {
|
||||
await fs.unlink(path.join(WorkDir, "agents", `${id}.json`));
|
||||
await fs.unlink(path.join(this.agentsDir, `${id}.md`));
|
||||
}
|
||||
}
|
||||
|
|
@ -21,6 +21,7 @@ import { IBus } from "../application/lib/bus.js";
|
|||
import { IMessageQueue } from "../application/lib/message-queue.js";
|
||||
import { IRunsRepo } from "../runs/repo.js";
|
||||
import { IRunsLock } from "../runs/lock.js";
|
||||
import { PrefixLogger } from "../shared/prefix-logger.js";
|
||||
|
||||
export interface IAgentRuntime {
|
||||
trigger(runId: string): Promise<void>;
|
||||
|
|
@ -63,6 +64,11 @@ export class AgentRuntime implements IAgentRuntime {
|
|||
return;
|
||||
}
|
||||
try {
|
||||
await this.bus.publish({
|
||||
runId,
|
||||
type: "run-processing-start",
|
||||
subflow: [],
|
||||
});
|
||||
while (true) {
|
||||
let eventCount = 0;
|
||||
const run = await this.runsRepo.fetch(runId);
|
||||
|
|
@ -94,6 +100,11 @@ export class AgentRuntime implements IAgentRuntime {
|
|||
}
|
||||
} finally {
|
||||
await this.runsLock.release(runId);
|
||||
await this.bus.publish({
|
||||
runId,
|
||||
type: "run-processing-end",
|
||||
subflow: [],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -499,6 +510,8 @@ export async function* streamAgent({
|
|||
messageQueue: IMessageQueue;
|
||||
modelConfigRepo: IModelConfigRepo;
|
||||
}): AsyncGenerator<z.infer<typeof RunEvent>, void, unknown> {
|
||||
const logger = new PrefixLogger(`run-${runId}-${state.agentName}`);
|
||||
|
||||
async function* processEvent(event: z.infer<typeof RunEvent>): AsyncGenerator<z.infer<typeof RunEvent>, void, unknown> {
|
||||
state.ingest(event);
|
||||
yield event;
|
||||
|
|
@ -518,61 +531,29 @@ export async function* streamAgent({
|
|||
// set up provider + model
|
||||
const provider = await getProvider(agent.provider);
|
||||
const model = provider.languageModel(agent.model || modelConfig.defaults.model);
|
||||
|
||||
let loopCounter = 0;
|
||||
|
||||
console.log('here');
|
||||
|
||||
async function pendingMsgs() {
|
||||
const pendingMsgs = [];
|
||||
|
||||
}
|
||||
|
||||
while (true) {
|
||||
// console.error(`loop counter: ${loopCounter++}`)
|
||||
// if last response is from assistant and text, get any pending msgs
|
||||
const lastMessage = state.messages[state.messages.length - 1];
|
||||
if (lastMessage
|
||||
&& lastMessage.role === "assistant"
|
||||
&& (typeof lastMessage.content === "string"
|
||||
|| !lastMessage.content.some(part => part.type === "tool-call")
|
||||
)
|
||||
) {
|
||||
let pending = 0;
|
||||
while(true) {
|
||||
const msg = await messageQueue.dequeue(runId);
|
||||
if (!msg) {
|
||||
break;
|
||||
}
|
||||
pending++;
|
||||
yield *processEvent({
|
||||
runId,
|
||||
type: "message",
|
||||
messageId: msg.messageId,
|
||||
message: {
|
||||
role: "user",
|
||||
content: msg.message,
|
||||
},
|
||||
subflow: [],
|
||||
});
|
||||
}
|
||||
// if no msgs found, return
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
loopCounter++;
|
||||
let loopLogger = logger.child(`iter-${loopCounter}`);
|
||||
loopLogger.log('starting loop iteration');
|
||||
|
||||
// execute any pending tool calls
|
||||
for (const toolCallId of Object.keys(state.pendingToolCalls)) {
|
||||
const toolCall = state.toolCallIdMap[toolCallId];
|
||||
let _logger = loopLogger.child(`tc-${toolCallId}-${toolCall.toolName}`);
|
||||
_logger.log('processing');
|
||||
|
||||
// if ask-human, skip
|
||||
if (toolCall.toolName === "ask-human") {
|
||||
_logger.log('skipping, reason: ask-human');
|
||||
continue;
|
||||
}
|
||||
|
||||
// if tool has been denied, deny
|
||||
if (state.deniedToolCallIds[toolCallId]) {
|
||||
yield *processEvent({
|
||||
_logger.log('returning denied tool message, reason: tool has been denied');
|
||||
yield* processEvent({
|
||||
runId,
|
||||
messageId: await idGenerator.next(),
|
||||
type: "message",
|
||||
|
|
@ -587,13 +568,15 @@ export async function* streamAgent({
|
|||
continue;
|
||||
}
|
||||
|
||||
// if permission is pending on this tool call, allow execution
|
||||
// if permission is pending on this tool call, skip execution
|
||||
if (state.pendingToolPermissionRequests[toolCallId]) {
|
||||
_logger.log('skipping, reason: permission is pending');
|
||||
continue;
|
||||
}
|
||||
|
||||
// execute approved tool
|
||||
yield *processEvent({
|
||||
_logger.log('executing tool');
|
||||
yield* processEvent({
|
||||
runId,
|
||||
type: "tool-invocation",
|
||||
toolCallId,
|
||||
|
|
@ -611,7 +594,7 @@ export async function* streamAgent({
|
|||
messageQueue,
|
||||
modelConfigRepo,
|
||||
})) {
|
||||
yield *processEvent({
|
||||
yield* processEvent({
|
||||
...event,
|
||||
subflow: [toolCallId, ...event.subflow],
|
||||
});
|
||||
|
|
@ -637,7 +620,7 @@ export async function* streamAgent({
|
|||
result: result,
|
||||
subflow: [],
|
||||
});
|
||||
yield *processEvent({
|
||||
yield* processEvent({
|
||||
runId,
|
||||
messageId: await idGenerator.next(),
|
||||
type: "message",
|
||||
|
|
@ -647,26 +630,20 @@ export async function* streamAgent({
|
|||
}
|
||||
}
|
||||
|
||||
// if pending state, exit
|
||||
// if waiting on user permission or ask-human, exit
|
||||
if (state.getPendingAskHumans().length || state.getPendingPermissions().length) {
|
||||
// console.error("pending asks or permissions, exiting (b.)")
|
||||
loopLogger.log('exiting loop, reason: pending asks or permissions');
|
||||
return;
|
||||
}
|
||||
|
||||
// if current message state isn't runnable, exit
|
||||
/*
|
||||
if (state.messages.length === 0 || state.messages[state.messages.length - 1].role === "assistant") {
|
||||
// console.error("current message state isn't runnable, exiting (c.)")
|
||||
return;
|
||||
}
|
||||
*/
|
||||
|
||||
while(true) {
|
||||
// get any queued user messages
|
||||
while (true) {
|
||||
const msg = await messageQueue.dequeue(runId);
|
||||
if (!msg) {
|
||||
break;
|
||||
}
|
||||
yield *processEvent({
|
||||
loopLogger.log('dequeued user message', msg.messageId);
|
||||
yield* processEvent({
|
||||
runId,
|
||||
type: "message",
|
||||
messageId: msg.messageId,
|
||||
|
|
@ -678,7 +655,20 @@ export async function* streamAgent({
|
|||
});
|
||||
}
|
||||
|
||||
// if last response is from assistant and text, exit
|
||||
const lastMessage = state.messages[state.messages.length - 1];
|
||||
if (lastMessage
|
||||
&& lastMessage.role === "assistant"
|
||||
&& (typeof lastMessage.content === "string"
|
||||
|| !lastMessage.content.some(part => part.type === "tool-call")
|
||||
)
|
||||
) {
|
||||
loopLogger.log('exiting loop, reason: last message is from assistant and text');
|
||||
return;
|
||||
}
|
||||
|
||||
// run one LLM turn.
|
||||
loopLogger.log('running llm turn');
|
||||
// stream agent response and build message
|
||||
const messageBuilder = new StreamStepMessageBuilder();
|
||||
for await (const event of streamLlm(
|
||||
|
|
@ -687,8 +677,9 @@ export async function* streamAgent({
|
|||
agent.instructions,
|
||||
tools,
|
||||
)) {
|
||||
loopLogger.log('got llm-stream-event:', event.type)
|
||||
messageBuilder.ingest(event);
|
||||
yield *processEvent({
|
||||
yield* processEvent({
|
||||
runId,
|
||||
type: "llm-stream-event",
|
||||
event: event,
|
||||
|
|
@ -698,7 +689,7 @@ export async function* streamAgent({
|
|||
|
||||
// build and emit final message from agent response
|
||||
const message = messageBuilder.get();
|
||||
yield *processEvent({
|
||||
yield* processEvent({
|
||||
runId,
|
||||
messageId: await idGenerator.next(),
|
||||
type: "message",
|
||||
|
|
@ -712,7 +703,8 @@ export async function* streamAgent({
|
|||
if (part.type === "tool-call") {
|
||||
const underlyingTool = agent.tools![part.toolName];
|
||||
if (underlyingTool.type === "builtin" && underlyingTool.name === "ask-human") {
|
||||
yield *processEvent({
|
||||
loopLogger.log('emitting ask-human-request, toolCallId:', part.toolCallId);
|
||||
yield* processEvent({
|
||||
runId,
|
||||
type: "ask-human-request",
|
||||
toolCallId: part.toolCallId,
|
||||
|
|
@ -723,7 +715,8 @@ export async function* streamAgent({
|
|||
if (underlyingTool.type === "builtin" && underlyingTool.name === "executeCommand") {
|
||||
// if command is blocked, then seek permission
|
||||
if (isBlocked(part.arguments.command)) {
|
||||
yield *processEvent({
|
||||
loopLogger.log('emitting tool-permission-request, toolCallId:', part.toolCallId);
|
||||
yield* processEvent({
|
||||
runId,
|
||||
type: "tool-permission-request",
|
||||
toolCall: part,
|
||||
|
|
@ -732,14 +725,15 @@ export async function* streamAgent({
|
|||
}
|
||||
}
|
||||
if (underlyingTool.type === "agent" && underlyingTool.name) {
|
||||
yield *processEvent({
|
||||
loopLogger.log('emitting spawn-subflow, toolCallId:', part.toolCallId);
|
||||
yield* processEvent({
|
||||
runId,
|
||||
type: "spawn-subflow",
|
||||
agentName: underlyingTool.name,
|
||||
toolCallId: part.toolCallId,
|
||||
subflow: [],
|
||||
});
|
||||
yield *processEvent({
|
||||
yield* processEvent({
|
||||
runId,
|
||||
messageId: await idGenerator.next(),
|
||||
type: "message",
|
||||
|
|
|
|||
|
|
@ -9,8 +9,7 @@ import { RunEvent } from "./entities/run-events.js";
|
|||
import { createInterface, Interface } from "node:readline/promises";
|
||||
import { ToolCallPart } from "./entities/message.js";
|
||||
import { Agent } from "./agents/agents.js";
|
||||
import { McpServerConfig } from "./mcp/mcp.js";
|
||||
import { McpServerDefinition } from "./mcp/mcp.js";
|
||||
import { McpServerConfig, McpServerDefinition } from "./mcp/schema.js";
|
||||
import { Example } from "./entities/example.js";
|
||||
import { z } from "zod";
|
||||
import { Flavor } from "./models/models.js";
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import { resolveSkill, availableSkills } from "../assistant/skills/index.js";
|
|||
import { executeTool, listServers, listTools } from "../../mcp/mcp.js";
|
||||
import container from "../../di/container.js";
|
||||
import { IMcpConfigRepo } from "../..//mcp/repo.js";
|
||||
import { McpServerDefinition } from "../../mcp/mcp.js";
|
||||
import { McpServerDefinition } from "../../mcp/schema.js";
|
||||
|
||||
const BuiltinToolsSchema = z.record(z.string(), z.object({
|
||||
description: z.string(),
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ export class InMemoryBus implements IBus {
|
|||
private subscribers: Map<string, ((event: z.infer<typeof RunEvent>) => Promise<void>)[]> = new Map();
|
||||
|
||||
async publish(event: z.infer<typeof RunEvent>): Promise<void> {
|
||||
console.log(this.subscribers);
|
||||
const pending: Promise<void>[] = [];
|
||||
for (const subscriber of this.subscribers.get(event.runId) || []) {
|
||||
pending.push(subscriber(event));
|
||||
|
|
@ -21,7 +20,6 @@ export class InMemoryBus implements IBus {
|
|||
for (const subscriber of this.subscribers.get('*') || []) {
|
||||
pending.push(subscriber(event));
|
||||
}
|
||||
console.log(pending.length);
|
||||
await Promise.all(pending);
|
||||
}
|
||||
|
||||
|
|
@ -30,7 +28,6 @@ export class InMemoryBus implements IBus {
|
|||
this.subscribers.set(runId, []);
|
||||
}
|
||||
this.subscribers.get(runId)!.push(handler);
|
||||
console.log(this.subscribers);
|
||||
return () => {
|
||||
this.subscribers.get(runId)!.splice(this.subscribers.get(runId)!.indexOf(handler), 1);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import z from "zod"
|
||||
import { Agent } from "../agents/agents.js"
|
||||
import { McpServerDefinition } from "../mcp/mcp.js";
|
||||
import { McpServerDefinition } from "../mcp/schema.js";
|
||||
|
||||
export const Example = z.object({
|
||||
id: z.string(),
|
||||
|
|
@ -9,4 +9,4 @@ export const Example = z.object({
|
|||
entryAgent: z.string().optional(),
|
||||
agents: z.array(Agent).optional(),
|
||||
mcpServers: z.record(z.string(), McpServerDefinition).optional(),
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -8,6 +8,14 @@ const BaseRunEvent = z.object({
|
|||
subflow: z.array(z.string()),
|
||||
});
|
||||
|
||||
export const RunProcessingStartEvent = BaseRunEvent.extend({
|
||||
type: z.literal("run-processing-start"),
|
||||
});
|
||||
|
||||
export const RunProcessingEndEvent = BaseRunEvent.extend({
|
||||
type: z.literal("run-processing-end"),
|
||||
});
|
||||
|
||||
export const StartEvent = BaseRunEvent.extend({
|
||||
type: z.literal("start"),
|
||||
agentName: z.string(),
|
||||
|
|
@ -73,6 +81,8 @@ export const RunErrorEvent = BaseRunEvent.extend({
|
|||
});
|
||||
|
||||
export const RunEvent = z.union([
|
||||
RunProcessingStartEvent,
|
||||
RunProcessingEndEvent,
|
||||
StartEvent,
|
||||
SpawnSubFlowEvent,
|
||||
LlmStreamEvent,
|
||||
|
|
|
|||
286
apps/cli/src/knowledge/sync_calendar.ts
Normal file
286
apps/cli/src/knowledge/sync_calendar.ts
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { google } from 'googleapis';
|
||||
import { authenticate } from '@google-cloud/local-auth';
|
||||
import { OAuth2Client } from 'google-auth-library';
|
||||
import { NodeHtmlMarkdown } from 'node-html-markdown'
|
||||
|
||||
// Configuration
|
||||
const CREDENTIALS_PATH = path.join(process.cwd(), 'credentials.json');
|
||||
const TOKEN_PATH = path.join(process.cwd(), 'token_calendar_notes.json'); // Changed to force re-auth with new scopes
|
||||
const SYNC_INTERVAL_MS = 60 * 1000;
|
||||
const SCOPES = [
|
||||
'https://www.googleapis.com/auth/calendar.readonly',
|
||||
'https://www.googleapis.com/auth/drive.readonly'
|
||||
];
|
||||
|
||||
const nhm = new NodeHtmlMarkdown();
|
||||
|
||||
// --- Auth Functions ---
|
||||
|
||||
async function loadSavedCredentialsIfExist(): Promise<OAuth2Client | null> {
|
||||
try {
|
||||
if (!fs.existsSync(TOKEN_PATH)) return null;
|
||||
const tokenContent = fs.readFileSync(TOKEN_PATH, 'utf-8');
|
||||
const tokenData = JSON.parse(tokenContent);
|
||||
|
||||
const credsContent = fs.readFileSync(CREDENTIALS_PATH, 'utf-8');
|
||||
const keys = JSON.parse(credsContent);
|
||||
const key = keys.installed || keys.web;
|
||||
|
||||
const client = new google.auth.OAuth2(
|
||||
key.client_id,
|
||||
key.client_secret,
|
||||
key.redirect_uris ? key.redirect_uris[0] : 'http://localhost'
|
||||
);
|
||||
|
||||
client.setCredentials({
|
||||
refresh_token: tokenData.refresh_token || tokenData.refreshToken,
|
||||
access_token: tokenData.token || tokenData.access_token,
|
||||
expiry_date: tokenData.expiry || tokenData.expiry_date,
|
||||
scope: tokenData.scope
|
||||
});
|
||||
|
||||
return client;
|
||||
} catch (err) {
|
||||
console.error("Error loading saved credentials:", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function saveCredentials(client: OAuth2Client) {
|
||||
const content = fs.readFileSync(CREDENTIALS_PATH, 'utf-8');
|
||||
const keys = JSON.parse(content);
|
||||
const key = keys.installed || keys.web;
|
||||
const payload = JSON.stringify({
|
||||
type: 'authorized_user',
|
||||
client_id: key.client_id,
|
||||
client_secret: key.client_secret,
|
||||
refresh_token: client.credentials.refresh_token,
|
||||
access_token: client.credentials.access_token,
|
||||
expiry_date: client.credentials.expiry_date,
|
||||
}, null, 2);
|
||||
fs.writeFileSync(TOKEN_PATH, payload);
|
||||
}
|
||||
|
||||
async function authorize(): Promise<OAuth2Client> {
|
||||
let client = await loadSavedCredentialsIfExist();
|
||||
if (client && client.credentials && client.credentials.expiry_date && client.credentials.expiry_date > Date.now()) {
|
||||
console.log("Using existing valid token.");
|
||||
return client;
|
||||
}
|
||||
|
||||
if (client && client.credentials && (!client.credentials.expiry_date || client.credentials.expiry_date <= Date.now()) && client.credentials.refresh_token) {
|
||||
console.log("Refreshing expired token...");
|
||||
try {
|
||||
await client.refreshAccessToken();
|
||||
await saveCredentials(client);
|
||||
return client;
|
||||
} catch (e) {
|
||||
console.error("Failed to refresh token:", e);
|
||||
if (fs.existsSync(TOKEN_PATH)) fs.unlinkSync(TOKEN_PATH);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("Performing new OAuth authentication...");
|
||||
client = await authenticate({
|
||||
scopes: SCOPES,
|
||||
keyfilePath: CREDENTIALS_PATH,
|
||||
}) as any;
|
||||
if (client && client.credentials) {
|
||||
await saveCredentials(client);
|
||||
}
|
||||
return client!;
|
||||
}
|
||||
|
||||
// --- Helper Functions ---
|
||||
|
||||
function cleanFilename(name: string): string {
|
||||
return name.replace(/[\\/*?:\"<>|]/g, "").replace(/\s+/g, "_").substring(0, 100).trim();
|
||||
}
|
||||
|
||||
// --- Sync Logic ---
|
||||
|
||||
function cleanUpOldFiles(currentEventIds: Set<string>, syncDir: string) {
|
||||
if (!fs.existsSync(syncDir)) return;
|
||||
|
||||
const files = fs.readdirSync(syncDir);
|
||||
for (const filename of files) {
|
||||
if (filename === 'sync_state.json') continue;
|
||||
|
||||
// We expect files like:
|
||||
// {eventId}.json
|
||||
// {eventId}_doc_{docId}.md
|
||||
|
||||
let eventId: string | null = null;
|
||||
|
||||
if (filename.endsWith('.json')) {
|
||||
eventId = filename.replace('.json', '');
|
||||
} else if (filename.endsWith('.md')) {
|
||||
// Try to extract eventId from prefix
|
||||
// Assuming eventId doesn't contain underscores usually, but if it does, this split might be fragile.
|
||||
// Google Calendar IDs are usually alphanumeric.
|
||||
// Let's rely on the delimiter we use: "_doc_"
|
||||
const parts = filename.split('_doc_');
|
||||
if (parts.length > 1) {
|
||||
eventId = parts[0];
|
||||
}
|
||||
}
|
||||
|
||||
if (eventId && !currentEventIds.has(eventId)) {
|
||||
try {
|
||||
fs.unlinkSync(path.join(syncDir, filename));
|
||||
console.log(`Removed old/out-of-window file: ${filename}`);
|
||||
} catch (e) {
|
||||
console.error(`Error deleting file ${filename}:`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function saveEvent(event: any, syncDir: string): Promise<boolean> {
|
||||
const eventId = event.id;
|
||||
if (!eventId) return false;
|
||||
|
||||
const filePath = path.join(syncDir, `${eventId}.json`);
|
||||
|
||||
try {
|
||||
fs.writeFileSync(filePath, JSON.stringify(event, null, 2));
|
||||
return true;
|
||||
} catch (e) {
|
||||
console.error(`Error saving event ${eventId}:`, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function processAttachments(drive: any, event: any, syncDir: string) {
|
||||
if (!event.attachments || event.attachments.length === 0) return;
|
||||
|
||||
const eventId = event.id;
|
||||
const eventTitle = event.summary || 'Untitled';
|
||||
const eventDate = event.start?.dateTime || event.start?.date || 'Unknown';
|
||||
const organizer = event.organizer?.email || 'Unknown';
|
||||
|
||||
for (const att of event.attachments) {
|
||||
// We only care about Google Docs
|
||||
if (att.mimeType === 'application/vnd.google-apps.document') {
|
||||
const fileId = att.fileId;
|
||||
const safeTitle = cleanFilename(att.title);
|
||||
// Unique filename linked to event
|
||||
const filename = `${eventId}_doc_${safeTitle}.md`;
|
||||
const filePath = path.join(syncDir, filename);
|
||||
|
||||
// Simple cache check: if file exists, skip.
|
||||
// Ideally we check modifiedTime, but that requires an extra API call per file.
|
||||
// Given the loop interval, we can just check existence to save quota.
|
||||
// If user updates notes, they might want them re-synced.
|
||||
// For now, let's just check existence. To be smarter, we'd need a state file or check API.
|
||||
if (fs.existsSync(filePath)) continue;
|
||||
|
||||
try {
|
||||
const res = await drive.files.export({
|
||||
fileId: fileId,
|
||||
mimeType: 'text/html'
|
||||
});
|
||||
|
||||
const html = res.data;
|
||||
const md = nhm.translate(html);
|
||||
|
||||
const frontmatter = [
|
||||
`# ${att.title}`,
|
||||
`**Event:** ${eventTitle}`,
|
||||
`**Date:** ${eventDate}`,
|
||||
`**Organizer:** ${organizer}`,
|
||||
`**Link:** ${att.fileUrl}`,
|
||||
`---`,
|
||||
``
|
||||
].join('\n');
|
||||
|
||||
fs.writeFileSync(filePath, frontmatter + md);
|
||||
console.log(`Synced Note: ${att.title} for event ${eventTitle}`);
|
||||
} catch (e) {
|
||||
console.error(`Failed to download note ${att.title}:`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function syncCalendarWindow(auth: OAuth2Client, syncDir: string, lookbackDays: number) {
|
||||
// Calculate window
|
||||
const now = new Date();
|
||||
const lookbackMs = lookbackDays * 24 * 60 * 60 * 1000;
|
||||
const twoWeeksForwardMs = 14 * 24 * 60 * 60 * 1000;
|
||||
|
||||
const timeMin = new Date(now.getTime() - lookbackMs).toISOString();
|
||||
const timeMax = new Date(now.getTime() + twoWeeksForwardMs).toISOString();
|
||||
|
||||
console.log(`Syncing calendar from ${timeMin} to ${timeMax} (lookback: ${lookbackDays} days)...`);
|
||||
|
||||
const calendar = google.calendar({ version: 'v3', auth });
|
||||
const drive = google.drive({ version: 'v3', auth });
|
||||
|
||||
try {
|
||||
const res = await calendar.events.list({
|
||||
calendarId: 'primary',
|
||||
timeMin: timeMin,
|
||||
timeMax: timeMax,
|
||||
singleEvents: true,
|
||||
orderBy: 'startTime'
|
||||
});
|
||||
|
||||
const events = res.data.items || [];
|
||||
const currentEventIds = new Set<string>();
|
||||
|
||||
if (events.length === 0) {
|
||||
console.log("No events found in this window.");
|
||||
} else {
|
||||
console.log(`Found ${events.length} events.`);
|
||||
for (const event of events) {
|
||||
if (event.id) {
|
||||
await saveEvent(event, syncDir);
|
||||
await processAttachments(drive, event, syncDir);
|
||||
currentEventIds.add(event.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cleanUpOldFiles(currentEventIds, syncDir);
|
||||
|
||||
} catch (error) {
|
||||
console.error("An error occurred during calendar sync:", error);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("Starting Google Calendar & Notes Sync (TS)...");
|
||||
|
||||
const syncDirArg = process.argv[2];
|
||||
const lookbackDaysArg = process.argv[3];
|
||||
|
||||
const SYNC_DIR = syncDirArg || 'synced_calendar_events';
|
||||
const LOOKBACK_DAYS = lookbackDaysArg ? parseInt(lookbackDaysArg, 10) : 14;
|
||||
|
||||
if (isNaN(LOOKBACK_DAYS) || LOOKBACK_DAYS <= 0) {
|
||||
console.error("Error: Lookback days must be a positive number.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(SYNC_DIR)) {
|
||||
fs.mkdirSync(SYNC_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
try {
|
||||
const auth = await authorize();
|
||||
console.log("Authorization successful.");
|
||||
|
||||
while (true) {
|
||||
await syncCalendarWindow(auth, SYNC_DIR, LOOKBACK_DAYS);
|
||||
console.log(`Sleeping for ${SYNC_INTERVAL_MS / 1000} seconds...`);
|
||||
await new Promise(resolve => setTimeout(resolve, SYNC_INTERVAL_MS));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Fatal error in main loop:", error);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
368
apps/cli/src/knowledge/sync_gmail.ts
Normal file
368
apps/cli/src/knowledge/sync_gmail.ts
Normal file
|
|
@ -0,0 +1,368 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { google } from 'googleapis';
|
||||
import { authenticate } from '@google-cloud/local-auth';
|
||||
import { NodeHtmlMarkdown } from 'node-html-markdown'
|
||||
import { OAuth2Client } from 'google-auth-library';
|
||||
|
||||
// Configuration
|
||||
const DEFAULT_SYNC_DIR = 'synced_emails_ts';
|
||||
const CREDENTIALS_PATH = path.join(process.cwd(), 'credentials.json');
|
||||
const TOKEN_PATH = path.join(process.cwd(), 'token_api.json'); // Reuse Python's token
|
||||
const SYNC_INTERVAL_MS = 60 * 1000;
|
||||
const SCOPES = ['https://www.googleapis.com/auth/gmail.readonly'];
|
||||
|
||||
const nhm = new NodeHtmlMarkdown();
|
||||
|
||||
// --- Auth Functions ---
|
||||
|
||||
async function loadSavedCredentialsIfExist(): Promise<OAuth2Client | null> {
|
||||
try {
|
||||
const tokenContent = fs.readFileSync(TOKEN_PATH, 'utf-8');
|
||||
const tokenData = JSON.parse(tokenContent);
|
||||
|
||||
const credsContent = fs.readFileSync(CREDENTIALS_PATH, 'utf-8');
|
||||
const keys = JSON.parse(credsContent);
|
||||
const key = keys.installed || keys.web;
|
||||
|
||||
// Manually construct credentials for google.auth.fromJSON
|
||||
const credentials = {
|
||||
type: 'authorized_user',
|
||||
client_id: key.client_id,
|
||||
client_secret: key.client_secret,
|
||||
refresh_token: tokenData.refresh_token || tokenData.refreshToken, // Handle both cases
|
||||
access_token: tokenData.token || tokenData.access_token, // Handle both cases
|
||||
expiry_date: tokenData.expiry || tokenData.expiry_date
|
||||
};
|
||||
return google.auth.fromJSON(credentials) as OAuth2Client;
|
||||
} catch (err) {
|
||||
console.error("Error loading saved credentials:", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function saveCredentials(client: OAuth2Client) {
|
||||
const content = fs.readFileSync(CREDENTIALS_PATH, 'utf-8');
|
||||
const keys = JSON.parse(content);
|
||||
const key = keys.installed || keys.web;
|
||||
const payload = JSON.stringify({
|
||||
type: 'authorized_user',
|
||||
client_id: key.client_id,
|
||||
client_secret: key.client_secret,
|
||||
refresh_token: client.credentials.refresh_token,
|
||||
access_token: client.credentials.access_token,
|
||||
expiry_date: client.credentials.expiry_date,
|
||||
}, null, 2);
|
||||
fs.writeFileSync(TOKEN_PATH, payload);
|
||||
}
|
||||
|
||||
async function authorize(): Promise<OAuth2Client> {
|
||||
let client = await loadSavedCredentialsIfExist();
|
||||
if (client && client.credentials && client.credentials.expiry_date && client.credentials.expiry_date > Date.now()) {
|
||||
console.log("Using existing valid token.");
|
||||
return client;
|
||||
}
|
||||
|
||||
if (client && client.credentials && (!client.credentials.expiry_date || client.credentials.expiry_date <= Date.now()) && client.credentials.refresh_token) {
|
||||
console.log("Refreshing expired token...");
|
||||
try {
|
||||
await client.refreshAccessToken();
|
||||
await saveCredentials(client); // Save refreshed token
|
||||
return client;
|
||||
} catch (e) {
|
||||
console.error("Failed to refresh token:", e);
|
||||
// Fall through to full re-auth if refresh fails
|
||||
fs.existsSync(TOKEN_PATH) && fs.unlinkSync(TOKEN_PATH);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("Performing new OAuth authentication...");
|
||||
client = await authenticate({
|
||||
scopes: SCOPES,
|
||||
keyfilePath: CREDENTIALS_PATH,
|
||||
}) as any;
|
||||
if (client && client.credentials) {
|
||||
await saveCredentials(client);
|
||||
}
|
||||
return client!;
|
||||
}
|
||||
|
||||
// --- Helper Functions ---
|
||||
|
||||
function cleanFilename(name: string): string {
|
||||
return name.replace(/[\\/*?:":<>|]/g, "").substring(0, 100).trim();
|
||||
}
|
||||
|
||||
function decodeBase64(data: string): string {
|
||||
return Buffer.from(data, 'base64').toString('utf-8');
|
||||
}
|
||||
|
||||
function getBody(payload: any): string {
|
||||
let body = "";
|
||||
if (payload.parts) {
|
||||
for (const part of payload.parts) {
|
||||
if (part.mimeType === 'text/plain' && part.body && part.body.data) {
|
||||
const text = decodeBase64(part.body.data);
|
||||
// Strip quoted lines
|
||||
const cleanLines = text.split('\n').filter((line: string) => !line.trim().startsWith('>'));
|
||||
body += cleanLines.join('\n');
|
||||
} else if (part.mimeType === 'text/html' && part.body && part.body.data) {
|
||||
const html = decodeBase64(part.body.data);
|
||||
let md = nhm.translate(html);
|
||||
// Simple quote stripping for MD
|
||||
const cleanLines = md.split('\n').filter((line: string) => !line.trim().startsWith('>'));
|
||||
body += cleanLines.join('\n');
|
||||
} else if (part.parts) {
|
||||
body += getBody(part);
|
||||
}
|
||||
}
|
||||
} else if (payload.body && payload.body.data) {
|
||||
const data = decodeBase64(payload.body.data);
|
||||
if (payload.mimeType === 'text/html') {
|
||||
let md = nhm.translate(data);
|
||||
body += md.split('\n').filter((line: string) => !line.trim().startsWith('>')).join('\n');
|
||||
} else {
|
||||
body += data.split('\n').filter((line: string) => !line.trim().startsWith('>')).join('\n');
|
||||
}
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
async function saveAttachment(gmail: any, userId: string, msgId: string, part: any, attachmentsDir: string): Promise<string | null> {
|
||||
const filename = part.filename;
|
||||
const attId = part.body?.attachmentId;
|
||||
if (!filename || !attId) return null;
|
||||
|
||||
const safeName = `${msgId}_${cleanFilename(filename)}`;
|
||||
const filePath = path.join(attachmentsDir, safeName);
|
||||
|
||||
if (fs.existsSync(filePath)) return safeName;
|
||||
|
||||
try {
|
||||
const res = await gmail.users.messages.attachments.get({
|
||||
userId,
|
||||
messageId: msgId,
|
||||
id: attId
|
||||
});
|
||||
|
||||
const data = res.data.data;
|
||||
if (data) {
|
||||
fs.writeFileSync(filePath, Buffer.from(data, 'base64'));
|
||||
console.log(`Saved attachment: ${safeName}`);
|
||||
return safeName;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Error saving attachment ${filename}:`, e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// --- Sync Logic ---
|
||||
|
||||
async function processThread(auth: OAuth2Client, threadId: string, syncDir: string, attachmentsDir: string) {
|
||||
const gmail = google.gmail({ version: 'v1', auth });
|
||||
try {
|
||||
const res = await gmail.users.threads.get({ userId: 'me', id: threadId });
|
||||
const thread = res.data;
|
||||
const messages = thread.messages;
|
||||
|
||||
if (!messages || messages.length === 0) return;
|
||||
|
||||
// Subject from first message
|
||||
const firstHeader = messages[0].payload?.headers;
|
||||
const subject = firstHeader?.find(h => h.name === 'Subject')?.value || '(No Subject)';
|
||||
|
||||
let mdContent = `# ${subject}\n\n`;
|
||||
mdContent += `**Thread ID:** ${threadId}\n`;
|
||||
mdContent += `**Message Count:** ${messages.length}\n\n---\n\n`;
|
||||
|
||||
for (const msg of messages) {
|
||||
const msgId = msg.id!;
|
||||
const headers = msg.payload?.headers || [];
|
||||
const from = headers.find(h => h.name === 'From')?.value || 'Unknown';
|
||||
const date = headers.find(h => h.name === 'Date')?.value || 'Unknown';
|
||||
|
||||
mdContent += `### From: ${from}\n`;
|
||||
mdContent += `**Date:** ${date}\n\n`;
|
||||
|
||||
const body = getBody(msg.payload);
|
||||
mdContent += `${body}\n\n`;
|
||||
|
||||
// Attachments
|
||||
const parts: any[] = [];
|
||||
const traverseParts = (pList: any[]) => {
|
||||
for (const p of pList) {
|
||||
parts.push(p);
|
||||
if (p.parts) traverseParts(p.parts);
|
||||
}
|
||||
};
|
||||
if (msg.payload?.parts) traverseParts(msg.payload.parts);
|
||||
|
||||
let attachmentsFound = false;
|
||||
for (const part of parts) {
|
||||
if (part.filename && part.body?.attachmentId) {
|
||||
const savedName = await saveAttachment(gmail, 'me', msgId, part, attachmentsDir);
|
||||
if (savedName) {
|
||||
if (!attachmentsFound) {
|
||||
mdContent += "**Attachments:**\n";
|
||||
attachmentsFound = true;
|
||||
}
|
||||
mdContent += `- [${part.filename}](attachments/${savedName})\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
mdContent += "\n---\n\n";
|
||||
}
|
||||
|
||||
fs.writeFileSync(path.join(syncDir, `${threadId}.md`), mdContent);
|
||||
console.log(`Synced Thread: ${subject} (${threadId})`);
|
||||
|
||||
} catch (error) {
|
||||
console.error(`Error processing thread ${threadId}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
function loadState(stateFile: string): { historyId?: string } {
|
||||
if (fs.existsSync(stateFile)) {
|
||||
return JSON.parse(fs.readFileSync(stateFile, 'utf-8'));
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
function saveState(historyId: string, stateFile: string) {
|
||||
fs.writeFileSync(stateFile, JSON.stringify({
|
||||
historyId,
|
||||
last_sync: new Date().toISOString()
|
||||
}, null, 2));
|
||||
}
|
||||
|
||||
async function fullSync(auth: OAuth2Client, syncDir: string, attachmentsDir: string, stateFile: string, lookbackDays: number) {
|
||||
console.log(`Performing full sync of last ${lookbackDays} days...`);
|
||||
const gmail = google.gmail({ version: 'v1', auth });
|
||||
|
||||
const pastDate = new Date();
|
||||
pastDate.setDate(pastDate.getDate() - lookbackDays);
|
||||
const dateQuery = pastDate.toISOString().split('T')[0].replace(/-/g, '/');
|
||||
|
||||
// Get History ID
|
||||
const profile = await gmail.users.getProfile({ userId: 'me' });
|
||||
const currentHistoryId = profile.data.historyId!;
|
||||
|
||||
let pageToken: string | undefined;
|
||||
do {
|
||||
const res: any = await gmail.users.threads.list({
|
||||
userId: 'me',
|
||||
q: `after:${dateQuery}`,
|
||||
pageToken
|
||||
});
|
||||
|
||||
const threads = res.data.threads;
|
||||
if (threads) {
|
||||
for (const thread of threads) {
|
||||
await processThread(auth, thread.id!, syncDir, attachmentsDir);
|
||||
}
|
||||
}
|
||||
pageToken = res.data.nextPageToken;
|
||||
} while (pageToken);
|
||||
|
||||
saveState(currentHistoryId, stateFile);
|
||||
console.log("Full sync complete.");
|
||||
}
|
||||
|
||||
async function partialSync(auth: OAuth2Client, startHistoryId: string, syncDir: string, attachmentsDir: string, stateFile: string, lookbackDays: number) {
|
||||
console.log(`Checking updates since historyId ${startHistoryId}...`);
|
||||
const gmail = google.gmail({ version: 'v1', auth });
|
||||
|
||||
try {
|
||||
const res = await gmail.users.history.list({
|
||||
userId: 'me',
|
||||
startHistoryId,
|
||||
historyTypes: ['messageAdded']
|
||||
});
|
||||
|
||||
const changes = res.data.history;
|
||||
if (!changes || changes.length === 0) {
|
||||
console.log("No new changes.");
|
||||
const profile = await gmail.users.getProfile({ userId: 'me' });
|
||||
saveState(profile.data.historyId!, stateFile);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Found ${changes.length} history records.`);
|
||||
const threadIds = new Set<string>();
|
||||
|
||||
for (const record of changes) {
|
||||
if (record.messagesAdded) {
|
||||
for (const item of record.messagesAdded) {
|
||||
if (item.message?.threadId) {
|
||||
threadIds.add(item.message.threadId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const tid of threadIds) {
|
||||
await processThread(auth, tid, syncDir, attachmentsDir);
|
||||
}
|
||||
|
||||
const profile = await gmail.users.getProfile({ userId: 'me' });
|
||||
saveState(profile.data.historyId!, stateFile);
|
||||
|
||||
} catch (error: any) {
|
||||
if (error.response?.status === 404) {
|
||||
console.log("History ID expired. Falling back to full sync.");
|
||||
await fullSync(auth, syncDir, attachmentsDir, stateFile, lookbackDays);
|
||||
} else {
|
||||
console.error("Error during partial sync:", error);
|
||||
// If 401, remove token to force re-auth next run
|
||||
if (error.response?.status === 401 && fs.existsSync(TOKEN_PATH)) {
|
||||
console.log("401 Unauthorized. Deleting token to force re-authentication.");
|
||||
fs.unlinkSync(TOKEN_PATH);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("Starting Gmail Sync (TS)...");
|
||||
const syncDirArg = process.argv[2];
|
||||
const lookbackDaysArg = process.argv[3];
|
||||
|
||||
const SYNC_DIR = syncDirArg || DEFAULT_SYNC_DIR;
|
||||
const LOOKBACK_DAYS = lookbackDaysArg ? parseInt(lookbackDaysArg, 10) : 7; // Default to 7 days
|
||||
|
||||
if (isNaN(LOOKBACK_DAYS) || LOOKBACK_DAYS <= 0) {
|
||||
console.error("Error: Lookback days must be a positive number.");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const ATTACHMENTS_DIR = path.join(SYNC_DIR, 'attachments');
|
||||
const STATE_FILE = path.join(SYNC_DIR, 'sync_state.json');
|
||||
|
||||
// Ensure directories exist
|
||||
if (!fs.existsSync(SYNC_DIR)) fs.mkdirSync(SYNC_DIR, { recursive: true });
|
||||
if (!fs.existsSync(ATTACHMENTS_DIR)) fs.mkdirSync(ATTACHMENTS_DIR, { recursive: true });
|
||||
|
||||
try {
|
||||
const auth = await authorize();
|
||||
console.log("Authorization successful.");
|
||||
|
||||
while (true) {
|
||||
const state = loadState(STATE_FILE);
|
||||
if (!state.historyId) {
|
||||
console.log("No history ID found, starting full sync...");
|
||||
await fullSync(auth, SYNC_DIR, ATTACHMENTS_DIR, STATE_FILE, LOOKBACK_DAYS);
|
||||
} else {
|
||||
console.log("History ID found, starting partial sync...");
|
||||
await partialSync(auth, state.historyId, SYNC_DIR, ATTACHMENTS_DIR, STATE_FILE, LOOKBACK_DAYS);
|
||||
}
|
||||
|
||||
console.log(`Sleeping for ${SYNC_INTERVAL_MS / 1000} seconds...`);
|
||||
await new Promise(resolve => setTimeout(resolve, SYNC_INTERVAL_MS));
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Fatal error in main loop:", error);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(console.error);
|
||||
|
|
@ -6,63 +6,12 @@ import z from "zod";
|
|||
import { IMcpConfigRepo } from "./repo.js";
|
||||
import { Transport } from "@modelcontextprotocol/sdk/shared/transport.js";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
||||
|
||||
export const StdioMcpServerConfig = z.object({
|
||||
type: z.literal("stdio").optional(),
|
||||
command: z.string(),
|
||||
args: z.array(z.string()).optional(),
|
||||
env: z.record(z.string(), z.string()).optional(),
|
||||
});
|
||||
|
||||
export const HttpMcpServerConfig = z.object({
|
||||
type: z.literal("http").optional(),
|
||||
url: z.string(),
|
||||
headers: z.record(z.string(), z.string()).optional(),
|
||||
});
|
||||
|
||||
export const McpServerDefinition = z.union([StdioMcpServerConfig, HttpMcpServerConfig]);
|
||||
|
||||
export const McpServerConfig = z.object({
|
||||
mcpServers: z.record(z.string(), McpServerDefinition),
|
||||
});
|
||||
|
||||
const connectionState = z.enum(["disconnected", "connected", "error"]);
|
||||
|
||||
export const McpServerList = z.object({
|
||||
mcpServers: z.record(z.string(), z.object({
|
||||
config: McpServerDefinition,
|
||||
state: connectionState,
|
||||
error: z.string().nullable(),
|
||||
})),
|
||||
});
|
||||
|
||||
/*
|
||||
inputSchema: {
|
||||
[x: string]: unknown;
|
||||
type: "object";
|
||||
properties?: Record<string, object> | undefined;
|
||||
required?: string[] | undefined;
|
||||
};
|
||||
*/
|
||||
export const Tool = z.object({
|
||||
name: z.string(),
|
||||
description: z.string().optional(),
|
||||
inputSchema: z.object({
|
||||
type: z.literal("object"),
|
||||
properties: z.record(z.string(), z.any()).optional(),
|
||||
required: z.array(z.string()).optional(),
|
||||
}),
|
||||
outputSchema: z.object({
|
||||
type: z.literal("object"),
|
||||
properties: z.record(z.string(), z.any()).optional(),
|
||||
required: z.array(z.string()).optional(),
|
||||
}).optional(),
|
||||
})
|
||||
|
||||
export const ListToolsResponse = z.object({
|
||||
tools: z.array(Tool),
|
||||
nextCursor: z.string().optional(),
|
||||
});
|
||||
import {
|
||||
connectionState,
|
||||
ListToolsResponse,
|
||||
McpServerDefinition,
|
||||
McpServerList,
|
||||
} from "./schema.js";
|
||||
|
||||
type mcpState = {
|
||||
state: z.infer<typeof connectionState>,
|
||||
|
|
@ -171,4 +120,4 @@ export async function executeTool(serverName: string, toolName: string, input: a
|
|||
arguments: input,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { WorkDir } from "../config/config.js";
|
||||
import { McpServerConfig } from "./mcp.js";
|
||||
import { McpServerDefinition } from "./mcp.js";
|
||||
import { McpServerConfig, McpServerDefinition } from "./schema.js";
|
||||
import fs from "fs/promises";
|
||||
import path from "path";
|
||||
import z from "zod";
|
||||
|
|
@ -42,4 +41,4 @@ export class FSMcpConfigRepo implements IMcpConfigRepo {
|
|||
delete conf.mcpServers[serverName];
|
||||
await fs.writeFile(this.configPath, JSON.stringify(conf, null, 2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
50
apps/cli/src/mcp/schema.ts
Normal file
50
apps/cli/src/mcp/schema.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import z from "zod";
|
||||
|
||||
export const StdioMcpServerConfig = z.object({
|
||||
type: z.literal("stdio").optional(),
|
||||
command: z.string(),
|
||||
args: z.array(z.string()).optional(),
|
||||
env: z.record(z.string(), z.string()).optional(),
|
||||
});
|
||||
|
||||
export const HttpMcpServerConfig = z.object({
|
||||
type: z.literal("http").optional(),
|
||||
url: z.string(),
|
||||
headers: z.record(z.string(), z.string()).optional(),
|
||||
});
|
||||
|
||||
export const McpServerDefinition = z.union([StdioMcpServerConfig, HttpMcpServerConfig]);
|
||||
|
||||
export const McpServerConfig = z.object({
|
||||
mcpServers: z.record(z.string(), McpServerDefinition),
|
||||
});
|
||||
|
||||
export const connectionState = z.enum(["disconnected", "connected", "error"]);
|
||||
|
||||
export const McpServerList = z.object({
|
||||
mcpServers: z.record(z.string(), z.object({
|
||||
config: McpServerDefinition,
|
||||
state: connectionState,
|
||||
error: z.string().nullable(),
|
||||
})),
|
||||
});
|
||||
|
||||
export const Tool = z.object({
|
||||
name: z.string(),
|
||||
description: z.string().optional(),
|
||||
inputSchema: z.object({
|
||||
type: z.literal("object"),
|
||||
properties: z.record(z.string(), z.any()).optional(),
|
||||
required: z.array(z.string()).optional(),
|
||||
}),
|
||||
outputSchema: z.object({
|
||||
type: z.literal("object"),
|
||||
properties: z.record(z.string(), z.any()).optional(),
|
||||
required: z.array(z.string()).optional(),
|
||||
}).optional(),
|
||||
});
|
||||
|
||||
export const ListToolsResponse = z.object({
|
||||
tools: z.array(Tool),
|
||||
nextCursor: z.string().optional(),
|
||||
});
|
||||
|
|
@ -4,8 +4,8 @@ import { streamSSE } from 'hono/streaming'
|
|||
import { describeRoute, validator, resolver, openAPIRouteHandler } from "hono-openapi"
|
||||
import z from 'zod';
|
||||
import container from './di/container.js';
|
||||
import { executeTool, listServers, listTools, ListToolsResponse, McpServerList } from "./mcp/mcp.js";
|
||||
import { McpServerDefinition } from "./mcp/mcp.js";
|
||||
import { executeTool, listServers, listTools } from "./mcp/mcp.js";
|
||||
import { ListToolsResponse, McpServerDefinition, McpServerList } from "./mcp/schema.js";
|
||||
import { IMcpConfigRepo } from './mcp/repo.js';
|
||||
import { IModelConfigRepo } from './models/repo.js';
|
||||
import { ModelConfig, Provider } from "./models/models.js";
|
||||
|
|
@ -14,6 +14,7 @@ import { Agent } from "./agents/agents.js";
|
|||
import { AskHumanResponsePayload, authorizePermission, createMessage, createRun, replyToHumanInputRequest, Run, stop, ToolPermissionAuthorizePayload } from './runs/runs.js';
|
||||
import { IRunsRepo, CreateRunOptions, ListRunsResponse } from './runs/repo.js';
|
||||
import { IBus } from './application/lib/bus.js';
|
||||
import { cors } from 'hono/cors';
|
||||
|
||||
let id = 0;
|
||||
|
||||
|
|
@ -620,7 +621,6 @@ const routes = new Hono()
|
|||
unsub = await bus.subscribe('*', async (event) => {
|
||||
if (aborted) return;
|
||||
|
||||
console.log('got ev', event);
|
||||
await stream.writeSSE({
|
||||
data: JSON.stringify(event),
|
||||
event: "message",
|
||||
|
|
@ -638,6 +638,7 @@ const routes = new Hono()
|
|||
;
|
||||
|
||||
const app = new Hono()
|
||||
.use("/*", cors())
|
||||
.route("/", routes)
|
||||
.get(
|
||||
"/openapi.json",
|
||||
|
|
@ -665,4 +666,4 @@ serve({
|
|||
// PUT /skills/<id>
|
||||
// DELETE /skills/<id>
|
||||
|
||||
// GET /sse
|
||||
// GET /sse
|
||||
|
|
|
|||
26
apps/cli/src/shared/prefix-logger.ts
Normal file
26
apps/cli/src/shared/prefix-logger.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
// create a PrefixLogger class that wraps console.log with a prefix
|
||||
// and allows chaining with a parent logger
|
||||
export class PrefixLogger {
|
||||
private prefix: string;
|
||||
private parent: PrefixLogger | null;
|
||||
|
||||
constructor(prefix: string, parent: PrefixLogger | null = null) {
|
||||
this.prefix = prefix;
|
||||
this.parent = parent;
|
||||
}
|
||||
|
||||
log(...args: any[]) {
|
||||
const timestamp = new Date().toISOString();
|
||||
const prefix = '[' + this.prefix + ']';
|
||||
|
||||
if (this.parent) {
|
||||
this.parent.log(prefix, ...args);
|
||||
} else {
|
||||
console.log(timestamp, prefix, ...args);
|
||||
}
|
||||
}
|
||||
|
||||
child(childPrefix: string): PrefixLogger {
|
||||
return new PrefixLogger(childPrefix, this);
|
||||
}
|
||||
}
|
||||
190
apps/cli/src/tui/api.ts
Normal file
190
apps/cli/src/tui/api.ts
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
import { createParser } from "eventsource-parser";
|
||||
import { Agent } from "../agents/agents.js";
|
||||
import { AskHumanResponsePayload, Run, ToolPermissionAuthorizePayload } from "../runs/runs.js";
|
||||
import { ListRunsResponse } from "../runs/repo.js";
|
||||
import { ModelConfig } from "../models/models.js";
|
||||
import { RunEvent } from "../entities/run-events.js";
|
||||
import z from "zod";
|
||||
|
||||
const HealthSchema = z.object({
|
||||
status: z.literal("ok"),
|
||||
});
|
||||
|
||||
const MessageResponse = z.object({
|
||||
messageId: z.string(),
|
||||
});
|
||||
|
||||
const SuccessSchema = z.object({
|
||||
success: z.literal(true),
|
||||
});
|
||||
|
||||
type RunEventType = z.infer<typeof RunEvent>;
|
||||
|
||||
export interface RowboatApiOptions {
|
||||
baseUrl?: string;
|
||||
}
|
||||
|
||||
export class RowboatApi {
|
||||
readonly baseUrl: string;
|
||||
constructor({ baseUrl }: RowboatApiOptions = {}) {
|
||||
this.baseUrl = baseUrl ?? process.env.ROWBOATX_SERVER_URL ?? "http://127.0.0.1:3000";
|
||||
}
|
||||
|
||||
private buildUrl(pathname: string): string {
|
||||
return new URL(pathname, this.baseUrl).toString();
|
||||
}
|
||||
|
||||
private async request<T>(pathname: string, init?: RequestInit): Promise<T> {
|
||||
const headers: Record<string, string> = {
|
||||
Accept: "application/json",
|
||||
};
|
||||
if (init?.headers instanceof Headers) {
|
||||
init.headers.forEach((value, key) => {
|
||||
headers[key] = value;
|
||||
});
|
||||
} else if (Array.isArray(init?.headers)) {
|
||||
for (const [key, value] of init.headers) {
|
||||
headers[key] = value;
|
||||
}
|
||||
} else if (init?.headers) {
|
||||
Object.assign(headers, init.headers as Record<string, string>);
|
||||
}
|
||||
if (init?.body && !headers["Content-Type"]) {
|
||||
headers["Content-Type"] = "application/json";
|
||||
}
|
||||
const response = await fetch(this.buildUrl(pathname), {
|
||||
method: "GET",
|
||||
...init,
|
||||
headers,
|
||||
});
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => "");
|
||||
throw new Error(`Request to ${pathname} failed (${response.status}): ${text || response.statusText}`);
|
||||
}
|
||||
if (response.status === 204) {
|
||||
return undefined as T;
|
||||
}
|
||||
const text = await response.text();
|
||||
if (!text) {
|
||||
return undefined as T;
|
||||
}
|
||||
return JSON.parse(text) as T;
|
||||
}
|
||||
|
||||
async getHealth(): Promise<z.infer<typeof HealthSchema>> {
|
||||
const payload = await this.request("/health");
|
||||
return HealthSchema.parse(payload);
|
||||
}
|
||||
|
||||
async getModelConfig(): Promise<z.infer<typeof ModelConfig>> {
|
||||
const payload = await this.request("/models");
|
||||
return ModelConfig.parse(payload);
|
||||
}
|
||||
|
||||
async listAgents(): Promise<z.infer<typeof Agent>[]> {
|
||||
const payload = await this.request("/agents");
|
||||
return Agent.array().parse(payload);
|
||||
}
|
||||
|
||||
async listRuns(cursor?: string): Promise<z.infer<typeof ListRunsResponse>> {
|
||||
const searchParams = new URLSearchParams();
|
||||
if (cursor) {
|
||||
searchParams.set("cursor", cursor);
|
||||
}
|
||||
const payload = await this.request(`/runs${searchParams.size ? `?${searchParams.toString()}` : ""}`);
|
||||
return ListRunsResponse.parse(payload);
|
||||
}
|
||||
|
||||
async getRun(runId: string): Promise<z.infer<typeof Run>> {
|
||||
const payload = await this.request(`/runs/${encodeURIComponent(runId)}`);
|
||||
return Run.parse(payload);
|
||||
}
|
||||
|
||||
async createRun(agentId: string): Promise<z.infer<typeof Run>> {
|
||||
const payload = await this.request("/runs/new", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ agentId }),
|
||||
});
|
||||
return Run.parse(payload);
|
||||
}
|
||||
|
||||
async sendMessage(runId: string, message: string): Promise<z.infer<typeof MessageResponse>> {
|
||||
const payload = await this.request(`/runs/${encodeURIComponent(runId)}/messages/new`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ message }),
|
||||
});
|
||||
return MessageResponse.parse(payload);
|
||||
}
|
||||
|
||||
async authorizeTool(runId: string, payload: z.infer<typeof ToolPermissionAuthorizePayload>): Promise<void> {
|
||||
const response = await this.request(`/runs/${encodeURIComponent(runId)}/permissions/authorize`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
SuccessSchema.parse(response);
|
||||
}
|
||||
|
||||
async replyToHuman(runId: string, requestId: string, payload: z.infer<typeof AskHumanResponsePayload>): Promise<void> {
|
||||
const response = await this.request(`/runs/${encodeURIComponent(runId)}/human-input-requests/${encodeURIComponent(requestId)}/reply`, {
|
||||
method: "POST",
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
SuccessSchema.parse(response);
|
||||
}
|
||||
|
||||
async stopRun(runId: string): Promise<void> {
|
||||
const response = await this.request(`/runs/${encodeURIComponent(runId)}/stop`, {
|
||||
method: "POST",
|
||||
});
|
||||
SuccessSchema.parse(response);
|
||||
}
|
||||
|
||||
async subscribeToEvents(onEvent: (event: RunEventType) => void, onError?: (error: Error) => void): Promise<() => void> {
|
||||
const controller = new AbortController();
|
||||
const response = await fetch(this.buildUrl("/stream"), {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Accept: "text/event-stream",
|
||||
},
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!response.ok || !response.body) {
|
||||
throw new Error(`Failed to subscribe to event stream (${response.status})`);
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
const parser = createParser((event) => {
|
||||
if (event.type !== "event" || !event.data) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const parsed = RunEvent.parse(JSON.parse(event.data));
|
||||
onEvent(parsed);
|
||||
} catch (error) {
|
||||
onError?.(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
});
|
||||
|
||||
(async () => {
|
||||
try {
|
||||
while (true) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
parser.feed(decoder.decode(value, { stream: true }));
|
||||
}
|
||||
} catch (error) {
|
||||
if (controller.signal.aborted) {
|
||||
return;
|
||||
}
|
||||
onError?.(error instanceof Error ? error : new Error(String(error)));
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
controller.abort();
|
||||
reader.cancel().catch(() => undefined);
|
||||
};
|
||||
}
|
||||
}
|
||||
8
apps/cli/src/tui/index.tsx
Normal file
8
apps/cli/src/tui/index.tsx
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
import React from "react";
|
||||
import { render } from "ink";
|
||||
import { RowboatTui } from "./ui.js";
|
||||
|
||||
export function runTui({ serverUrl }: { serverUrl?: string }) {
|
||||
const baseUrl = serverUrl ?? process.env.ROWBOATX_SERVER_URL ?? "http://127.0.0.1:3000";
|
||||
render(<RowboatTui serverUrl={baseUrl} />);
|
||||
}
|
||||
1174
apps/cli/src/tui/ui.tsx
Normal file
1174
apps/cli/src/tui/ui.tsx
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -11,6 +11,7 @@
|
|||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"sourceMap": true,
|
||||
"jsx": "react-jsx",
|
||||
"paths": {
|
||||
"@/*": [
|
||||
"./src/*"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue