mirror of
https://github.com/katanemo/plano.git
synced 2026-06-20 15:28:07 +02:00
fix(skills): honor skills-only orchestrator decisions, dedupe runtime helpers, warn on dropped picks
Addresses the code-review findings on 7f5bf641:
- Honor skills-only decisions: RouteDecision.route_name is now Option<String> and the orchestrator emits a decision when routes is empty but skills is non-empty. The LLM handler falls back to the originally-requested model and still injects activated skill bodies, matching the contract in docs/source/resources/skills.rst.
- Warn on allow-list misses: resolve_for_route now returns a SkillResolution that splits drops into "not allow-listed for this route" vs "not in catalog (hallucinated)". brightstaff logs each bucket so misconfigured routing_preferences[].skills lists become visible instead of vanishing silently.
- Consolidate runtime: common::skills_runtime is now the single source of truth (referenced_skills_catalog, resolve_for_route, resolve_selected_skills, augment_system_prompt_with_skills). brightstaff drops its local re-implementations and calls into common.
- Tests: 11 new tests in common::skills_runtime (catalog union, allow-list intersection, dedup, hallucination handling, XML escaping, body size cap) and 6 new tests in brightstaff::handlers::llm::model_selection cover inject_activated_skills_into_request, including the first-system-message rule and the Parts->Text flatten — both now documented on the function.
- Cap skill body size at 32 KiB with a UTF-8-safe tail-trim + marker so an oversized SKILL.md cannot blow the downstream context window.
- XML-escape skill name and base_dir in the <skill_content> wrapper as defense-in-depth (names are validated upstream, but the wrapper sits inside the system prompt).
- Bound find_project_root at \$HOME plus a 30-parent depth cap so CLI invocations outside HOME no longer walk to /.
This commit is contained in:
Spherrrical
parent
7f5bf641bb
commit
5e8d27fd3c
4 changed files with 639 additions and 139 deletions
|
|
@ -4,45 +4,117 @@
|
|||
//! crate) so they can be unit-tested on the native target without dragging
|
||||
//! in proxy-wasm host-call symbols or tokio runtime dependencies.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::configuration::{SkillRef, TopLevelRoutingPreference};
|
||||
|
||||
/// Filter `skills` down to the subset attached to `route_name` via
|
||||
/// `routing_preferences[].skills`. When the selected route has no `skills:`
|
||||
/// list, returns an empty vector — skills are scoped to routes, not global.
|
||||
///
|
||||
/// `routing_preferences` is the source of truth for which skills are even
|
||||
/// eligible for the orchestrator to activate on a given route.
|
||||
pub fn skills_for_route<'a>(
|
||||
skills: &'a [SkillRef],
|
||||
routing_preferences: &[TopLevelRoutingPreference],
|
||||
route_name: &str,
|
||||
) -> Vec<&'a SkillRef> {
|
||||
let Some(route) = routing_preferences.iter().find(|p| p.name == route_name) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Some(allow) = route.skills.as_ref() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut out: Vec<&SkillRef> = Vec::with_capacity(allow.len());
|
||||
for name in allow {
|
||||
if let Some(skill) = skills.iter().find(|s| &s.name == name) {
|
||||
out.push(skill);
|
||||
/// Upper bound on the byte length of a single skill body the runtime will
|
||||
/// inject into an upstream system prompt. SKILL.md files are typically a few
|
||||
/// kilobytes; this guard keeps a single oversized or malicious skill from
|
||||
/// blowing the downstream model's context window. Bodies longer than this
|
||||
/// are tail-trimmed with a marker line. ~32 KiB ≈ 8K tokens at the
|
||||
/// 4-bytes-per-token heuristic used elsewhere in brightstaff.
|
||||
pub const MAX_SKILL_BODY_BYTES: usize = 32 * 1024;
|
||||
|
||||
const SKILL_BODY_TRUNCATION_MARKER: &str = "\n…[skill body truncated]\n";
|
||||
|
||||
/// Outcome of resolving a list of orchestrator-selected skill names against
|
||||
/// a route's `skills:` allow-list and the runtime catalog. Callers should
|
||||
/// emit `warn!` for each name in `dropped_not_allowed` / `dropped_unknown`
|
||||
/// so misconfigured allow-lists and hallucinated picks are observable.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SkillResolution<'a> {
|
||||
/// Skills that survived both the allow-list and catalog filters, in
|
||||
/// orchestrator-selected order with duplicates removed.
|
||||
pub activated: Vec<&'a SkillRef>,
|
||||
/// Names the orchestrator selected that are NOT in the chosen route's
|
||||
/// `skills:` allow-list. Typically a cross-route mention — the model
|
||||
/// pulled a skill name from the global catalog that this route did not
|
||||
/// expose. Callers should `warn!`.
|
||||
pub dropped_not_allowed: Vec<String>,
|
||||
/// Names that ARE allow-listed for the route but are missing from the
|
||||
/// runtime catalog (skill removed / never installed / hallucinated).
|
||||
pub dropped_unknown: Vec<String>,
|
||||
}
|
||||
|
||||
/// Build the orchestrator-visible skills catalog from the union of every
|
||||
/// skill name referenced under `routing_preferences[].skills`. Skills not
|
||||
/// referenced by any route are excluded — they would just clutter the
|
||||
/// `<skills>` block with no way for the orchestrator to attach them. The
|
||||
/// result preserves `catalog` order and is deduplicated by name.
|
||||
pub fn referenced_skills_catalog(
|
||||
catalog: &[SkillRef],
|
||||
routes: &HashMap<String, TopLevelRoutingPreference>,
|
||||
) -> Vec<SkillRef> {
|
||||
let mut referenced: HashSet<&str> = HashSet::new();
|
||||
for route in routes.values() {
|
||||
if let Some(names) = route.skills.as_ref() {
|
||||
for name in names {
|
||||
referenced.insert(name.as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut out: Vec<SkillRef> = Vec::new();
|
||||
let mut seen: HashSet<String> = HashSet::new();
|
||||
for skill in catalog {
|
||||
if referenced.contains(skill.name.as_str()) && seen.insert(skill.name.clone()) {
|
||||
out.push(skill.clone());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Resolve a list of orchestrator-selected skill names to their `SkillRef`s.
|
||||
/// Unknown names are dropped silently — the orchestrator can hallucinate.
|
||||
/// Results are deduplicated by name, preserving the order Plano-Orchestrator
|
||||
/// returned.
|
||||
/// Filter `selected` skill names to those that are both (a) allow-listed
|
||||
/// for the chosen route via `route_allowlist` and (b) present in `catalog`,
|
||||
/// preserving orchestrator order and deduplicating. Drops are reported on
|
||||
/// the `SkillResolution` struct so callers can `warn!` and surface
|
||||
/// misconfiguration without re-walking the lists.
|
||||
pub fn resolve_for_route<'a>(
|
||||
catalog: &'a [SkillRef],
|
||||
route_allowlist: &[String],
|
||||
selected: &[String],
|
||||
) -> SkillResolution<'a> {
|
||||
let allowed: HashSet<&str> = route_allowlist.iter().map(String::as_str).collect();
|
||||
let mut activated: Vec<&SkillRef> = Vec::with_capacity(selected.len());
|
||||
let mut taken: HashSet<&str> = HashSet::new();
|
||||
let mut dropped_not_allowed: Vec<String> = Vec::new();
|
||||
let mut dropped_unknown: Vec<String> = Vec::new();
|
||||
for name in selected {
|
||||
if !taken.insert(name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
if !allowed.contains(name.as_str()) {
|
||||
dropped_not_allowed.push(name.clone());
|
||||
continue;
|
||||
}
|
||||
match catalog.iter().find(|s| &s.name == name) {
|
||||
Some(skill) => activated.push(skill),
|
||||
None => dropped_unknown.push(name.clone()),
|
||||
}
|
||||
}
|
||||
SkillResolution {
|
||||
activated,
|
||||
dropped_not_allowed,
|
||||
dropped_unknown,
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a list of orchestrator-selected skill names to their `SkillRef`s
|
||||
/// directly against the catalog, without any per-route allow-list. Use this
|
||||
/// for the "skills-only" path documented in `docs/source/resources/skills.rst`
|
||||
/// where the orchestrator returns skills but no route — the catalog itself
|
||||
/// (already pre-filtered to skills referenced by SOME route via
|
||||
/// `referenced_skills_catalog`) is the effective allow-list. Unknown names
|
||||
/// are dropped silently; results are deduplicated by name preserving order.
|
||||
pub fn resolve_selected_skills<'a>(
|
||||
skills: &'a [SkillRef],
|
||||
selected_names: &[String],
|
||||
) -> Vec<&'a SkillRef> {
|
||||
let mut out: Vec<&SkillRef> = Vec::with_capacity(selected_names.len());
|
||||
let mut seen: HashSet<&str> = HashSet::new();
|
||||
for name in selected_names {
|
||||
if out.iter().any(|s| &s.name == name) {
|
||||
if !seen.insert(name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
if let Some(skill) = skills.iter().find(|s| &s.name == name) {
|
||||
|
|
@ -53,12 +125,22 @@ pub fn resolve_selected_skills<'a>(
|
|||
}
|
||||
|
||||
/// Append the bodies of activated skills to a system prompt, wrapped in
|
||||
/// `<skill_content name="...">` tags so a future context-management pass can
|
||||
/// recognize and recompact them.
|
||||
/// `<skill_content name="..." [base_dir="..."]>…</skill_content>` tags so a
|
||||
/// future context-management pass can recognize and recompact them.
|
||||
///
|
||||
/// Returns `None` only if no base system prompt was supplied and no skills
|
||||
/// were activated. When skills are present the wrapper text always appears so
|
||||
/// the downstream model receives a clear, well-structured instruction block.
|
||||
/// Behavior contract (relied on by `brightstaff::handlers::llm::model_selection`):
|
||||
///
|
||||
/// * Returns `None` only when no base prompt was supplied **and** no skills
|
||||
/// were activated. Otherwise always returns `Some`.
|
||||
/// * The base prompt (if any) is kept verbatim and the skill block is
|
||||
/// appended after a blank line.
|
||||
/// * Each skill body is tail-trimmed at `MAX_SKILL_BODY_BYTES` bytes (UTF-8
|
||||
/// boundary safe) with a truncation marker, so a single oversized
|
||||
/// SKILL.md cannot blow the downstream context window.
|
||||
/// * `name` and `base_dir` are XML-attribute-escaped (`&`, `<`, `>`, `"`)
|
||||
/// so a maliciously named skill cannot break out of the wrapper. Skill
|
||||
/// names are already validated upstream, but defense-in-depth matters
|
||||
/// here because the wrapper is part of the LLM's system prompt.
|
||||
pub fn augment_system_prompt_with_skills(
|
||||
base_system_prompt: Option<String>,
|
||||
activated_skills: &[&SkillRef],
|
||||
|
|
@ -80,22 +162,66 @@ pub fn augment_system_prompt_with_skills(
|
|||
against each skill's base directory.\n\n",
|
||||
);
|
||||
for skill in activated_skills {
|
||||
buf.push_str(&format!("<skill_content name=\"{}\"", skill.name));
|
||||
buf.push_str(&format!(
|
||||
"<skill_content name=\"{}\"",
|
||||
xml_attr_escape(&skill.name)
|
||||
));
|
||||
if let Some(base_dir) = skill.base_dir.as_deref() {
|
||||
buf.push_str(&format!(" base_dir=\"{}\"", base_dir));
|
||||
buf.push_str(&format!(" base_dir=\"{}\"", xml_attr_escape(base_dir)));
|
||||
}
|
||||
buf.push_str(">\n");
|
||||
if let Some(body) = skill.body.as_deref() {
|
||||
buf.push_str(body.trim_end());
|
||||
buf.push_str(&truncate_skill_body(body));
|
||||
buf.push('\n');
|
||||
} else {
|
||||
buf.push_str(&format!("(skill description) {}\n", skill.description));
|
||||
buf.push_str(&format!(
|
||||
"(skill description) {}\n",
|
||||
xml_attr_escape(&skill.description)
|
||||
));
|
||||
}
|
||||
buf.push_str("</skill_content>\n\n");
|
||||
}
|
||||
Some(buf.trim_end().to_string())
|
||||
}
|
||||
|
||||
/// Escape a string for use inside an XML attribute value (double-quoted).
|
||||
/// Quotes `&`, `<`, `>`, and `"`; leaves single quotes alone since the
|
||||
/// wrapper always uses double quotes.
|
||||
fn xml_attr_escape(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for ch in s.chars() {
|
||||
match ch {
|
||||
'&' => out.push_str("&"),
|
||||
'<' => out.push_str("<"),
|
||||
'>' => out.push_str(">"),
|
||||
'"' => out.push_str("""),
|
||||
_ => out.push(ch),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Tail-trim `body` to at most `MAX_SKILL_BODY_BYTES` bytes, respecting
|
||||
/// UTF-8 character boundaries. Appends a marker so the downstream model
|
||||
/// can tell content was dropped. Pass-through for short bodies.
|
||||
fn truncate_skill_body(body: &str) -> String {
|
||||
let trimmed = body.trim_end();
|
||||
if trimmed.len() <= MAX_SKILL_BODY_BYTES {
|
||||
return trimmed.to_string();
|
||||
}
|
||||
// Reserve room for the marker so the final length is still within the
|
||||
// budget even when the marker is added.
|
||||
let budget = MAX_SKILL_BODY_BYTES.saturating_sub(SKILL_BODY_TRUNCATION_MARKER.len());
|
||||
let mut end = budget;
|
||||
while end > 0 && !trimmed.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
let mut out = String::with_capacity(end + SKILL_BODY_TRUNCATION_MARKER.len());
|
||||
out.push_str(&trimmed[..end]);
|
||||
out.push_str(SKILL_BODY_TRUNCATION_MARKER);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -126,49 +252,93 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
fn routes_map(
|
||||
routes: Vec<TopLevelRoutingPreference>,
|
||||
) -> HashMap<String, TopLevelRoutingPreference> {
|
||||
routes.into_iter().map(|r| (r.name.clone(), r)).collect()
|
||||
}
|
||||
|
||||
// --- referenced_skills_catalog ---
|
||||
|
||||
#[test]
|
||||
fn skills_for_route_returns_attached_skills() {
|
||||
fn referenced_catalog_is_union_across_routes() {
|
||||
let catalog = vec![
|
||||
skill("pdf-processing", "extract"),
|
||||
skill("pdf", "extract"),
|
||||
skill("code-review", "review"),
|
||||
skill("never-used", "x"),
|
||||
];
|
||||
let routes = vec![
|
||||
route("code review", Some(vec!["code-review"])),
|
||||
route("doc work", Some(vec!["pdf-processing"])),
|
||||
];
|
||||
let resolved = skills_for_route(&catalog, &routes, "code review");
|
||||
assert_eq!(resolved.len(), 1);
|
||||
assert_eq!(resolved[0].name, "code-review");
|
||||
let routes = routes_map(vec![
|
||||
route("docs", Some(vec!["pdf"])),
|
||||
route("review", Some(vec!["code-review"])),
|
||||
route("other", None),
|
||||
]);
|
||||
let out = referenced_skills_catalog(&catalog, &routes);
|
||||
let names: Vec<_> = out.iter().map(|s| s.name.as_str()).collect();
|
||||
assert!(names.contains(&"pdf"));
|
||||
assert!(names.contains(&"code-review"));
|
||||
assert!(!names.contains(&"never-used"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skills_for_route_empty_when_route_has_no_skills_list() {
|
||||
let catalog = vec![skill("pdf-processing", "extract")];
|
||||
let routes = vec![route("code review", None)];
|
||||
let resolved = skills_for_route(&catalog, &routes, "code review");
|
||||
assert!(resolved.is_empty());
|
||||
fn referenced_catalog_deduplicates_when_multiple_routes_share_a_skill() {
|
||||
let catalog = vec![skill("pdf", "extract")];
|
||||
let routes = routes_map(vec![
|
||||
route("a", Some(vec!["pdf"])),
|
||||
route("b", Some(vec!["pdf"])),
|
||||
]);
|
||||
let out = referenced_skills_catalog(&catalog, &routes);
|
||||
assert_eq!(out.len(), 1);
|
||||
}
|
||||
|
||||
// --- resolve_for_route ---
|
||||
|
||||
#[test]
|
||||
fn resolve_for_route_keeps_allowlisted_skills_in_orchestrator_order() {
|
||||
let catalog = vec![skill("a", ""), skill("b", ""), skill("c", "")];
|
||||
let allow = vec!["a".to_string(), "b".to_string(), "c".to_string()];
|
||||
let selected = vec!["c".to_string(), "a".to_string()];
|
||||
let r = resolve_for_route(&catalog, &allow, &selected);
|
||||
let names: Vec<_> = r.activated.iter().map(|s| s.name.as_str()).collect();
|
||||
assert_eq!(names, vec!["c", "a"]);
|
||||
assert!(r.dropped_not_allowed.is_empty());
|
||||
assert!(r.dropped_unknown.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skills_for_route_empty_when_route_missing() {
|
||||
let catalog = vec![skill("pdf-processing", "extract")];
|
||||
let routes = vec![route("code review", Some(vec!["pdf-processing"]))];
|
||||
let resolved = skills_for_route(&catalog, &routes, "no-such-route");
|
||||
assert!(resolved.is_empty());
|
||||
fn resolve_for_route_drops_cross_route_skill_into_not_allowed() {
|
||||
let catalog = vec![skill("pdf", ""), skill("payment", "")];
|
||||
let allow = vec!["pdf".to_string()]; // route only allows pdf
|
||||
let selected = vec!["pdf".to_string(), "payment".to_string()];
|
||||
let r = resolve_for_route(&catalog, &allow, &selected);
|
||||
assert_eq!(r.activated.len(), 1);
|
||||
assert_eq!(r.activated[0].name, "pdf");
|
||||
assert_eq!(r.dropped_not_allowed, vec!["payment".to_string()]);
|
||||
assert!(r.dropped_unknown.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skills_for_route_drops_unknown_skill_names() {
|
||||
let catalog = vec![skill("pdf-processing", "extract")];
|
||||
let routes = vec![route(
|
||||
"code review",
|
||||
Some(vec!["pdf-processing", "ghost-skill"]),
|
||||
)];
|
||||
let resolved = skills_for_route(&catalog, &routes, "code review");
|
||||
assert_eq!(resolved.len(), 1);
|
||||
assert_eq!(resolved[0].name, "pdf-processing");
|
||||
fn resolve_for_route_drops_hallucinated_skill_into_unknown() {
|
||||
let catalog = vec![skill("pdf", "")];
|
||||
let allow = vec!["pdf".to_string(), "imaginary".to_string()];
|
||||
let selected = vec!["pdf".to_string(), "imaginary".to_string()];
|
||||
let r = resolve_for_route(&catalog, &allow, &selected);
|
||||
assert_eq!(r.activated.len(), 1);
|
||||
assert_eq!(r.activated[0].name, "pdf");
|
||||
assert!(r.dropped_not_allowed.is_empty());
|
||||
assert_eq!(r.dropped_unknown, vec!["imaginary".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_for_route_deduplicates_repeats() {
|
||||
let catalog = vec![skill("pdf", "")];
|
||||
let allow = vec!["pdf".to_string()];
|
||||
let selected = vec!["pdf".to_string(), "pdf".to_string(), "pdf".to_string()];
|
||||
let r = resolve_for_route(&catalog, &allow, &selected);
|
||||
assert_eq!(r.activated.len(), 1);
|
||||
}
|
||||
|
||||
// --- resolve_selected_skills (skills-only path) ---
|
||||
|
||||
#[test]
|
||||
fn resolve_selected_skills_drops_unknown_and_dedupes() {
|
||||
let catalog = vec![
|
||||
|
|
@ -187,6 +357,8 @@ mod tests {
|
|||
assert_eq!(resolved[1].name, "pdf-processing");
|
||||
}
|
||||
|
||||
// --- augment_system_prompt_with_skills ---
|
||||
|
||||
#[test]
|
||||
fn augment_passthrough_with_no_skills() {
|
||||
let augmented = augment_system_prompt_with_skills(Some("you are helpful".to_string()), &[]);
|
||||
|
|
@ -212,4 +384,32 @@ mod tests {
|
|||
assert!(augmented.contains("<skill_content name=\"code-review\""));
|
||||
assert!(augmented.contains("look at diffs"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn augment_xml_escapes_skill_name_and_base_dir() {
|
||||
let mut s = skill("safe-name", "body");
|
||||
s.name = "bad\"name".to_string();
|
||||
s.base_dir = Some("/path/with\"quote".to_string());
|
||||
let augmented = augment_system_prompt_with_skills(None, &[&s]).expect("augmented");
|
||||
// Raw double-quote must NOT appear inside the attribute value — only
|
||||
// its escaped form. Otherwise it would close the attribute and let a
|
||||
// skill name inject arbitrary attributes / break out of the wrapper.
|
||||
assert!(augmented.contains("name=\"bad"name\""));
|
||||
assert!(augmented.contains("base_dir=\"/path/with"quote\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn augment_truncates_oversized_skill_body() {
|
||||
let big_body: String = "a".repeat(MAX_SKILL_BODY_BYTES * 2);
|
||||
let s = skill("huge", &big_body);
|
||||
let augmented = augment_system_prompt_with_skills(None, &[&s]).expect("augmented");
|
||||
// Truncation marker is present, so the body did NOT pass through verbatim.
|
||||
assert!(augmented.contains("[skill body truncated]"));
|
||||
// And the body slice cannot be longer than MAX_SKILL_BODY_BYTES + a
|
||||
// little wrapper overhead — definitely not 2× the cap.
|
||||
let body_section_end = augmented.find("</skill_content>").unwrap();
|
||||
let body_section_start = augmented.find(">\n").unwrap() + 2;
|
||||
let body_len = body_section_end - body_section_start;
|
||||
assert!(body_len <= MAX_SKILL_BODY_BYTES + 64);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue