add Plano agent skills framework and rule set (#797)

* feat: add initial documentation for Plano Agent Skills

* feat: readme with examples

* feat: add detailed skills documentation and examples for Plano

---------

Co-authored-by: Adil Hafeez <adil.hafeez@gmail.com>

skills/src/build.ts

@@ -0,0 +1,262 @@
+#!/usr/bin/env node
+
+import { readFileSync, writeFileSync, readdirSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+type Section = {
+  prefix: string;
+  number: number;
+  title: string;
+  description: string;
+};
+
+type Rule = {
+  file: string;
+  title: string;
+  impact: string;
+  impactDescription: string;
+  tags: string[];
+  body: string;
+  section: Section;
+};
+
+type ParsedFrontmatter = {
+  frontmatter: Record<string, string>;
+  body: string;
+};
+
+type Metadata = {
+  abstract: string;
+  version: string;
+  organization: string;
+};
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const RULES_DIR = join(__dirname, "..", "rules");
+const OUTPUT_FILE = join(__dirname, "..", "AGENTS.md");
+const METADATA_FILE = join(__dirname, "..", "metadata.json");
+
+const SECTIONS: Section[] = [
+  {
+    prefix: "config-",
+    number: 1,
+    title: "Configuration Fundamentals",
+    description:
+      "Core config.yaml structure, versioning, listener types, and provider setup — the entry point for every Plano deployment.",
+  },
+  {
+    prefix: "routing-",
+    number: 2,
+    title: "Routing & Model Selection",
+    description:
+      "Intelligent LLM routing using preferences, aliases, and defaults to match tasks to the best model.",
+  },
+  {
+    prefix: "agent-",
+    number: 3,
+    title: "Agent Orchestration",
+    description:
+      "Multi-agent patterns, agent descriptions, and orchestration strategies for building agentic applications.",
+  },
+  {
+    prefix: "filter-",
+    number: 4,
+    title: "Filter Chains & Guardrails",
+    description:
+      "Request/response processing pipelines — ordering, MCP integration, and safety guardrails.",
+  },
+  {
+    prefix: "observe-",
+    number: 5,
+    title: "Observability & Debugging",
+    description:
+      "OpenTelemetry tracing, log levels, span attributes, and sampling for production visibility.",
+  },
+  {
+    prefix: "cli-",
+    number: 6,
+    title: "CLI Operations",
+    description:
+      "Using the planoai CLI for startup, tracing, CLI agents, project init, and code generation.",
+  },
+  {
+    prefix: "deploy-",
+    number: 7,
+    title: "Deployment & Security",
+    description:
+      "Docker deployment, environment variable management, health checks, and state storage for production.",
+  },
+  {
+    prefix: "advanced-",
+    number: 8,
+    title: "Advanced Patterns",
+    description:
+      "Prompt targets, external API integration, rate limiting, and multi-listener architectures.",
+  },
+];
+
+function parseFrontmatter(content: string): ParsedFrontmatter | null {
+  const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+  if (!match) return null;
+
+  const frontmatter: Record<string, string> = {};
+  const lines = match[1].split("\n");
+  for (const line of lines) {
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) continue;
+    const key = line.slice(0, colonIdx).trim();
+    const value = line.slice(colonIdx + 1).trim();
+    frontmatter[key] = value;
+  }
+
+  return {
+    frontmatter,
+    body: match[2].trim(),
+  };
+}
+
+function inferSection(filename: string): Section | null {
+  for (const section of SECTIONS) {
+    if (filename.startsWith(section.prefix)) {
+      return section;
+    }
+  }
+  return null;
+}
+
+function main(): void {
+  const metadata = JSON.parse(readFileSync(METADATA_FILE, "utf-8")) as Metadata;
+
+  const files = readdirSync(RULES_DIR)
+    .filter((f) => f.endsWith(".md") && !f.startsWith("_"))
+    .sort();
+
+  const sectionRules = new Map<number, Rule[]>();
+  for (const section of SECTIONS) {
+    sectionRules.set(section.number, []);
+  }
+
+  let parseErrors = 0;
+
+  for (const file of files) {
+    const content = readFileSync(join(RULES_DIR, file), "utf-8");
+    const parsed = parseFrontmatter(content);
+
+    if (!parsed) {
+      console.error(`ERROR: Could not parse frontmatter in ${file}`);
+      parseErrors++;
+      continue;
+    }
+
+    const section = inferSection(file);
+    if (!section) {
+      console.warn(`WARN: No section found for ${file} — skipping`);
+      continue;
+    }
+
+    const rule: Rule = {
+      file,
+      title: parsed.frontmatter.title ?? file,
+      impact: parsed.frontmatter.impact ?? "MEDIUM",
+      impactDescription: parsed.frontmatter.impactDescription ?? "",
+      tags: parsed.frontmatter.tags
+        ? parsed.frontmatter.tags.split(",").map((t) => t.trim())
+        : [],
+      body: parsed.body,
+      section,
+    };
+    sectionRules.get(section.number)?.push(rule);
+  }
+
+  if (parseErrors > 0) {
+    console.error(`\nBuild failed: ${parseErrors} file(s) had parse errors.`);
+    process.exit(1);
+  }
+
+  for (const [, rules] of sectionRules) {
+    rules.sort((a, b) => a.title.localeCompare(b.title));
+  }
+
+  const lines: string[] = [];
+  lines.push(`# Plano Agent Skills`);
+  lines.push(``);
+  lines.push(`> ${metadata.abstract}`);
+  lines.push(``);
+  lines.push(
+    `**Version:** ${metadata.version} | **Organization:** ${metadata.organization}`
+  );
+  lines.push(``);
+  lines.push(`---`);
+  lines.push(``);
+
+  lines.push(`## Table of Contents`);
+  lines.push(``);
+  for (const section of SECTIONS) {
+    const rules = sectionRules.get(section.number) ?? [];
+    if (rules.length === 0) continue;
+    lines.push(
+      `- [Section ${section.number}: ${section.title}](#section-${section.number})`
+    );
+    for (let i = 0; i < rules.length; i++) {
+      const rule = rules[i];
+      const id = `${section.number}.${i + 1}`;
+      const anchor = rule.title
+        .toLowerCase()
+        .replace(/[^a-z0-9\s-]/g, "")
+        .replace(/\s+/g, "-");
+      lines.push(`  - [${id} ${rule.title}](#${anchor})`);
+    }
+  }
+  lines.push(``);
+  lines.push(`---`);
+  lines.push(``);
+
+  for (const section of SECTIONS) {
+    const rules = sectionRules.get(section.number) ?? [];
+    if (rules.length === 0) continue;
+
+    lines.push(`## Section ${section.number}: ${section.title}`);
+    lines.push(``);
+    lines.push(`*${section.description}*`);
+    lines.push(``);
+
+    for (let i = 0; i < rules.length; i++) {
+      const rule = rules[i];
+      const id = `${section.number}.${i + 1}`;
+
+      lines.push(`### ${id} ${rule.title}`);
+      lines.push(``);
+      lines.push(
+        `**Impact:** \`${rule.impact}\`${rule.impactDescription ? ` — ${rule.impactDescription}` : ""}`
+      );
+      if (rule.tags.length > 0) {
+        lines.push(`**Tags:** ${rule.tags.map((t) => `\`${t}\``).join(", ")}`);
+      }
+      lines.push(``);
+      lines.push(rule.body);
+      lines.push(``);
+      lines.push(`---`);
+      lines.push(``);
+    }
+  }
+
+  lines.push(`*Generated from individual rule files in \`rules/\`.*`);
+  lines.push(
+    `*To contribute, see [CONTRIBUTING](https://github.com/katanemo/archgw/blob/main/CONTRIBUTING.md).*`
+  );
+
+  writeFileSync(OUTPUT_FILE, lines.join("\n"), "utf-8");
+
+  let totalRules = 0;
+  for (const section of SECTIONS) {
+    const rules = sectionRules.get(section.number) ?? [];
+    if (rules.length > 0) {
+      console.log(`  Section ${section.number}: ${rules.length} rules`);
+      totalRules += rules.length;
+    }
+  }
+  console.log(`\nBuilt AGENTS.md with ${totalRules} rules.`);
+}
+
+main();

skills/src/extract-tests.ts

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+
+import { readFileSync, writeFileSync, readdirSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+type ParsedFrontmatter = {
+  frontmatter: Record<string, string>;
+  body: string;
+};
+
+type SectionPrefix = {
+  prefix: string;
+  number: number;
+  title: string;
+};
+
+type ExampleExtraction = {
+  incorrect: string | null;
+  correct: string | null;
+};
+
+type TestCaseEntry = {
+  id: string;
+  section: number;
+  sectionTitle: string;
+  title: string;
+  impact: string;
+  tags: string[];
+  testCase: {
+    description: string;
+    input: string | null;
+    expected: string | null;
+    evaluationPrompt: string;
+  };
+};
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const RULES_DIR = join(__dirname, "..", "rules");
+const OUTPUT_FILE = join(__dirname, "..", "test-cases.json");
+
+const SECTION_PREFIXES: SectionPrefix[] = [
+  { prefix: "config-", number: 1, title: "Configuration Fundamentals" },
+  { prefix: "routing-", number: 2, title: "Routing & Model Selection" },
+  { prefix: "agent-", number: 3, title: "Agent Orchestration" },
+  { prefix: "filter-", number: 4, title: "Filter Chains & Guardrails" },
+  { prefix: "observe-", number: 5, title: "Observability & Debugging" },
+  { prefix: "cli-", number: 6, title: "CLI Operations" },
+  { prefix: "deploy-", number: 7, title: "Deployment & Security" },
+  { prefix: "advanced-", number: 8, title: "Advanced Patterns" },
+];
+
+function parseFrontmatter(content: string): ParsedFrontmatter | null {
+  const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+  if (!match) return null;
+
+  const frontmatter: Record<string, string> = {};
+  const lines = match[1].split("\n");
+  for (const line of lines) {
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) continue;
+    const key = line.slice(0, colonIdx).trim();
+    const value = line.slice(colonIdx + 1).trim();
+    frontmatter[key] = value;
+  }
+
+  return { frontmatter, body: match[2].trim() };
+}
+
+function extractCodeBlocks(text: string): string[] {
+  const blocks: string[] = [];
+  const regex = /```(?:yaml|bash|python|typescript|json|sh)?\n([\s\S]*?)```/g;
+  let match: RegExpExecArray | null;
+  do {
+    match = regex.exec(text);
+    if (match) {
+      blocks.push(match[1].trim());
+    }
+  } while (match !== null);
+  return blocks;
+}
+
+function extractExamples(body: string): ExampleExtraction {
+  const incorrectMatch = body.match(
+    /\*\*Incorrect[^*]*\*\*[:\s]*([\s\S]*?)(?=\*\*Correct|\*\*Key|$)/
+  );
+  const correctMatch = body.match(
+    /\*\*Correct[^*]*\*\*[:\s]*([\s\S]*?)(?=\*\*Incorrect|\*\*Key|\*\*Note|Reference:|$)/
+  );
+
+  return {
+    incorrect: incorrectMatch
+      ? extractCodeBlocks(incorrectMatch[1]).join("\n\n")
+      : null,
+    correct: correctMatch ? extractCodeBlocks(correctMatch[1]).join("\n\n") : null,
+  };
+}
+
+function inferSection(filename: string): SectionPrefix | null {
+  for (const s of SECTION_PREFIXES) {
+    if (filename.startsWith(s.prefix)) return s;
+  }
+  return null;
+}
+
+function main(): void {
+  const files = readdirSync(RULES_DIR)
+    .filter((f) => f.endsWith(".md") && !f.startsWith("_"))
+    .sort();
+
+  const testCases: TestCaseEntry[] = [];
+
+  for (const file of files) {
+    const content = readFileSync(join(RULES_DIR, file), "utf-8");
+    const parsed = parseFrontmatter(content);
+    if (!parsed) continue;
+
+    const { frontmatter, body } = parsed;
+    const section = inferSection(file);
+    if (!section) continue;
+
+    const { incorrect, correct } = extractExamples(body);
+    if (!incorrect && !correct) continue;
+
+    testCases.push({
+      id: file.replace(".md", ""),
+      section: section.number,
+      sectionTitle: section.title,
+      title: frontmatter.title ?? file,
+      impact: frontmatter.impact ?? "MEDIUM",
+      tags: frontmatter.tags
+        ? frontmatter.tags.split(",").map((t) => t.trim())
+        : [],
+      testCase: {
+        description: `Detect and fix: "${frontmatter.title}"`,
+        input: incorrect,
+        expected: correct,
+        evaluationPrompt: `Given the following Plano config or CLI usage, identify if it violates the rule "${frontmatter.title}" and explain how to fix it.`,
+      },
+    });
+  }
+
+  writeFileSync(OUTPUT_FILE, JSON.stringify(testCases, null, 2), "utf-8");
+  console.log(`Extracted ${testCases.length} test cases to test-cases.json`);
+}
+
+main();

skills/src/validate.ts

@@ -0,0 +1,156 @@
+#!/usr/bin/env node
+
+import { readFileSync, readdirSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+type ParsedFrontmatter = {
+  frontmatter: Record<string, string>;
+  body: string;
+};
+
+type ValidationResult = {
+  errors: string[];
+  warnings: string[];
+};
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const RULES_DIR = join(__dirname, "..", "rules");
+
+const VALID_IMPACTS = [
+  "CRITICAL",
+  "HIGH",
+  "MEDIUM-HIGH",
+  "MEDIUM",
+  "LOW-MEDIUM",
+  "LOW",
+] as const;
+
+const SECTION_PREFIXES = [
+  "config-",
+  "routing-",
+  "agent-",
+  "filter-",
+  "observe-",
+  "cli-",
+  "deploy-",
+  "advanced-",
+];
+
+function parseFrontmatter(content: string): ParsedFrontmatter | null {
+  const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+  if (!match) return null;
+
+  const frontmatter: Record<string, string> = {};
+  const lines = match[1].split("\n");
+  for (const line of lines) {
+    const colonIdx = line.indexOf(":");
+    if (colonIdx === -1) continue;
+    const key = line.slice(0, colonIdx).trim();
+    const value = line.slice(colonIdx + 1).trim();
+    frontmatter[key] = value;
+  }
+
+  return { frontmatter, body: match[2].trim() };
+}
+
+function validateFile(file: string, content: string): ValidationResult {
+  const errors: string[] = [];
+  const warnings: string[] = [];
+
+  const parsed = parseFrontmatter(content);
+  if (!parsed) {
+    errors.push("Missing or malformed frontmatter (expected --- ... ---)");
+    return { errors, warnings };
+  }
+
+  const { frontmatter, body } = parsed;
+
+  if (!frontmatter.title) {
+    errors.push("Missing required frontmatter field: title");
+  }
+  if (!frontmatter.impact) {
+    errors.push("Missing required frontmatter field: impact");
+  } else if (!VALID_IMPACTS.includes(frontmatter.impact as (typeof VALID_IMPACTS)[number])) {
+    errors.push(
+      `Invalid impact value: "${frontmatter.impact}". Valid values: ${VALID_IMPACTS.join(", ")}`
+    );
+  }
+  if (!frontmatter.tags) {
+    warnings.push("No tags defined — consider adding relevant tags");
+  }
+
+  const hasValidPrefix = SECTION_PREFIXES.some((p) => file.startsWith(p));
+  if (!hasValidPrefix) {
+    errors.push(
+      `Filename must start with a valid prefix: ${SECTION_PREFIXES.join(", ")}`
+    );
+  }
+
+  if (body.length < 100) {
+    warnings.push("Rule body seems very short — consider adding more detail");
+  }
+
+  if (!body.includes("```")) {
+    warnings.push(
+      "No code examples found — rules should include YAML or CLI examples"
+    );
+  }
+
+  if (!body.includes("Incorrect") || !body.includes("Correct")) {
+    warnings.push(
+      "Consider adding both Incorrect and Correct examples for clarity"
+    );
+  }
+
+  return { errors, warnings };
+}
+
+function main(): void {
+  const files = readdirSync(RULES_DIR)
+    .filter((f) => f.endsWith(".md") && !f.startsWith("_"))
+    .sort();
+
+  let totalErrors = 0;
+  let totalWarnings = 0;
+  let filesWithIssues = 0;
+
+  console.log(`Validating ${files.length} rule files...\n`);
+
+  for (const file of files) {
+    const content = readFileSync(join(RULES_DIR, file), "utf-8");
+    const { errors, warnings } = validateFile(file, content);
+
+    if (errors.length > 0 || warnings.length > 0) {
+      filesWithIssues++;
+      console.log(`📄 ${file}`);
+
+      for (const error of errors) {
+        console.log(`  ❌ ERROR: ${error}`);
+        totalErrors++;
+      }
+      for (const warning of warnings) {
+        console.log(`  ⚠️  WARN:  ${warning}`);
+        totalWarnings++;
+      }
+      console.log();
+    } else {
+      console.log(`✅ ${file}`);
+    }
+  }
+
+  console.log(`\n--- Validation Summary ---`);
+  console.log(`Files checked:    ${files.length}`);
+  console.log(`Files with issues: ${filesWithIssues}`);
+  console.log(`Errors:           ${totalErrors}`);
+  console.log(`Warnings:         ${totalWarnings}`);
+
+  if (totalErrors > 0) {
+    console.log(`\nValidation FAILED with ${totalErrors} error(s).`);
+    process.exit(1);
+  } else {
+    console.log(`\nValidation passed.`);
+  }
+}
+
+main();