ktx/packages/cli/src/context/scan/relationship-benchmark-report.ts

import { isKtxRelationshipBenchmarkTuningEligible } from './relationship-benchmarks.js';
import type {
  KtxRelationshipBenchmarkCaseResult,
  KtxRelationshipBenchmarkFixture,
  KtxRelationshipBenchmarkMode,
  KtxRelationshipBenchmarkSuiteResult,
} from './relationship-benchmarks.js';

export type KtxRelationshipBenchmarkReportCaseStatus = 'run' | 'validation_blocked' | 'not_run';

export interface KtxRelationshipBenchmarkReportCase {
  fixtureId: string;
  fixtureName: string;
  tier: string;
  mode: KtxRelationshipBenchmarkMode;
  status: KtxRelationshipBenchmarkReportCaseStatus;
  reason: string | null;
  tuningEligible: boolean;
  metrics: {
    pkRecall: number | null;
    fkRecall: number | null;
    acceptedOrReviewRecall: number | null;
    acceptedFalsePositiveCount: number | null;
    sqlQueries: number | null;
    llmCalls: number | null;
    runtimeSeconds: number | null;
  };
  falsePositives: {
    pk: string[];
    fk: string[];
  };
  falseNegatives: {
    pk: string[];
    fk: string[];
  };
  skippedComposite: {
    pk: string[];
    fk: string[];
  };
}

export interface KtxRelationshipBenchmarkReport {
  generatedAt: string;
  headline: {
    caseCount: number;
    headlineCaseCount: number;
    headlinePkRecall: number;
    headlineFkRecall: number;
    headlineAcceptedOrReviewRecall: number;
    acceptedFalsePositiveCount: number;
    validationBlockedCount: number;
  };
  cases: KtxRelationshipBenchmarkReportCase[];
}

function key(fixtureId: string, mode: KtxRelationshipBenchmarkMode): string {
  return `${fixtureId}:${mode}`;
}

function fixed(value: number | null): string {
  return value === null ? '-' : value.toFixed(3);
}

function reportCaseReason(input: {
  fixture: KtxRelationshipBenchmarkFixture;
  result: KtxRelationshipBenchmarkCaseResult;
}): string | null {
  if (input.result.validationBlocked) {
    return 'validation unavailable for this benchmark mode';
  }

  if (input.fixture.validationBudget !== undefined && input.result.predicted.reviewFk.length > 0) {
    return `review candidate validation reasons: validation_unattempted (${input.result.predicted.reviewFk.length})`;
  }

  return null;
}

function reportCaseFromResult(input: {
  fixture: KtxRelationshipBenchmarkFixture;
  mode: KtxRelationshipBenchmarkMode;
  result: KtxRelationshipBenchmarkCaseResult;
}): KtxRelationshipBenchmarkReportCase {
  const status = input.result.validationBlocked ? 'validation_blocked' : 'run';
  return {
    fixtureId: input.fixture.id,
    fixtureName: input.fixture.name,
    tier: input.fixture.tier,
    mode: input.mode,
    status,
    reason: reportCaseReason({ fixture: input.fixture, result: input.result }),
    tuningEligible: isKtxRelationshipBenchmarkTuningEligible({
      fixture: input.fixture,
      mode: input.mode,
      validationBlocked: input.result.validationBlocked,
    }),
    metrics: {
      pkRecall: input.result.metrics.pkRecall,
      fkRecall: input.result.metrics.fkRecall,
      acceptedOrReviewRecall: input.result.metrics.acceptedOrReviewRecall,
      acceptedFalsePositiveCount: input.result.metrics.acceptedFalsePositiveCount,
      sqlQueries: input.result.metrics.sqlQueries,
      llmCalls: input.result.metrics.llmCalls,
      runtimeSeconds: input.result.metrics.runtimeSeconds,
    },
    falsePositives: input.result.falsePositives,
    falseNegatives: input.result.falseNegatives,
    skippedComposite: input.result.skippedComposite,
  };
}

function notRunCase(input: {
  fixture: KtxRelationshipBenchmarkFixture;
  mode: KtxRelationshipBenchmarkMode;
  reason: string;
}): KtxRelationshipBenchmarkReportCase {
  return {
    fixtureId: input.fixture.id,
    fixtureName: input.fixture.name,
    tier: input.fixture.tier,
    mode: input.mode,
    status: 'not_run',
    reason: input.reason,
    tuningEligible: false,
    metrics: {
      pkRecall: null,
      fkRecall: null,
      acceptedOrReviewRecall: null,
      acceptedFalsePositiveCount: null,
      sqlQueries: null,
      llmCalls: null,
      runtimeSeconds: null,
    },
    falsePositives: { pk: [], fk: [] },
    falseNegatives: { pk: [], fk: [] },
    skippedComposite: { pk: [], fk: [] },
  };
}

export function buildKtxRelationshipBenchmarkReport(input: {
  fixtures: readonly KtxRelationshipBenchmarkFixture[];
  suite: KtxRelationshipBenchmarkSuiteResult;
  modes: readonly KtxRelationshipBenchmarkMode[];
  generatedAt?: string;
}): KtxRelationshipBenchmarkReport {
  const resultsByKey = new Map(input.suite.cases.map((result) => [key(result.fixtureId, result.mode), result]));
  const cases: KtxRelationshipBenchmarkReportCase[] = [];

  for (const fixture of input.fixtures) {
    const selectedModes = new Set(fixture.defaultModes);
    for (const mode of input.modes) {
      const result = resultsByKey.get(key(fixture.id, mode));
      if (result) {
        cases.push(reportCaseFromResult({ fixture, mode, result }));
        continue;
      }
      cases.push(
        notRunCase({
          fixture,
          mode,
          reason: selectedModes.has(mode) ? 'mode produced no benchmark result' : 'mode not selected by fixture defaultModes',
        }),
      );
    }
  }

  return {
    generatedAt: input.generatedAt ?? new Date().toISOString(),
    headline: {
      caseCount: input.suite.aggregate.caseCount,
      headlineCaseCount: input.suite.aggregate.headlineCaseCount,
      headlinePkRecall: input.suite.aggregate.headlinePkRecall,
      headlineFkRecall: input.suite.aggregate.headlineFkRecall,
      headlineAcceptedOrReviewRecall: input.suite.aggregate.headlineAcceptedOrReviewRecall,
      acceptedFalsePositiveCount: input.suite.cases.reduce(
        (sum, result) => sum + result.metrics.acceptedFalsePositiveCount,
        0,
      ),
      validationBlockedCount: input.suite.validationBlockedCases.length,
    },
    cases,
  };
}

type KtxRelationshipBenchmarkFailureSelector = (
  item: KtxRelationshipBenchmarkReportCase,
) => readonly string[];

function sortedFailureLines(input: {
  cases: readonly KtxRelationshipBenchmarkReportCase[];
  select: KtxRelationshipBenchmarkFailureSelector;
}): string[] {
  return input.cases
    .flatMap((item) =>
      input.select(item).map((value) => ({
        fixtureId: item.fixtureId,
        mode: item.mode,
        status: item.status,
        value,
      })),
    )
    .sort((left, right) => {
      const leftKey = `${left.fixtureId}:${left.mode}:${left.status}:${left.value}`;
      const rightKey = `${right.fixtureId}:${right.mode}:${right.status}:${right.value}`;
      return leftKey.localeCompare(rightKey);
    })
    .map((item) => `- \`${item.fixtureId}\` / \`${item.mode}\` / \`${item.status}\`: ${item.value}`);
}

function failureBlock(input: {
  title: string;
  cases: readonly KtxRelationshipBenchmarkReportCase[];
  select: KtxRelationshipBenchmarkFailureSelector;
}): string[] {
  const values = sortedFailureLines({ cases: input.cases, select: input.select });
  return ['', `### ${input.title}`, '', ...(values.length > 0 ? values : ['- none'])];
}

function headlineFailureContextBlocks(report: KtxRelationshipBenchmarkReport): string[] {
  const headlineCases = report.cases.filter((item) => item.tuningEligible);
  const remainingPkMisses = sortedFailureLines({
    cases: headlineCases,
    select: (item) => item.falseNegatives.pk,
  });
  const remainingFkMisses = sortedFailureLines({
    cases: headlineCases,
    select: (item) => item.falseNegatives.fk,
  });

  return [
    '',
    '## Headline Failure Context',
    '',
    'Remaining headline misses after this run are listed here so recall gains and still-open algorithmic gaps are visible in the regenerated evidence report.',
    '',
    `- Remaining headline false-negative PKs: ${remainingPkMisses.length}`,
    `- Remaining headline false-negative FKs: ${remainingFkMisses.length}`,
    '',
    '### Remaining Headline False Negative PKs',
    '',
    ...(remainingPkMisses.length > 0 ? remainingPkMisses : ['- none']),
    '',
    '### Remaining Headline False Negative FKs',
    '',
    ...(remainingFkMisses.length > 0 ? remainingFkMisses : ['- none']),
  ];
}

function failureDetailBlocks(report: KtxRelationshipBenchmarkReport): string[] {
  const headlineCases = report.cases.filter((item) => item.tuningEligible);
  const otherCases = report.cases.filter((item) => !item.tuningEligible);

  return [
    '',
    '## Failure Details',
    ...failureBlock({
      title: 'Headline False Positive PKs',
      cases: headlineCases,
      select: (item) => item.falsePositives.pk,
    }),
    ...failureBlock({
      title: 'Headline False Positive FKs',
      cases: headlineCases,
      select: (item) => item.falsePositives.fk,
    }),
    ...failureBlock({
      title: 'Headline False Negative PKs',
      cases: headlineCases,
      select: (item) => item.falseNegatives.pk,
    }),
    ...failureBlock({
      title: 'Headline False Negative FKs',
      cases: headlineCases,
      select: (item) => item.falseNegatives.fk,
    }),
    ...failureBlock({
      title: 'Other False Positive PKs',
      cases: otherCases,
      select: (item) => item.falsePositives.pk,
    }),
    ...failureBlock({
      title: 'Other False Positive FKs',
      cases: otherCases,
      select: (item) => item.falsePositives.fk,
    }),
    ...failureBlock({
      title: 'Other False Negative PKs',
      cases: otherCases,
      select: (item) => item.falseNegatives.pk,
    }),
    ...failureBlock({
      title: 'Other False Negative FKs',
      cases: otherCases,
      select: (item) => item.falseNegatives.fk,
    }),
  ];
}

function compositeSkipBlocks(report: KtxRelationshipBenchmarkReport): string[] {
  const headlineCases = report.cases.filter((item) => item.tuningEligible);

  return [
    '',
    '## Composite Ground Truth Skips',
    ...failureBlock({
      title: 'Skipped Composite PKs',
      cases: headlineCases,
      select: (item) => item.skippedComposite.pk,
    }),
    ...failureBlock({
      title: 'Skipped Composite FKs',
      cases: headlineCases,
      select: (item) => item.skippedComposite.fk,
    }),
  ];
}

export function formatKtxRelationshipBenchmarkReportMarkdown(report: KtxRelationshipBenchmarkReport): string {
  const lines = [
    '# KTX Relationship Discovery Benchmark Evidence',
    '',
    `Generated: ${report.generatedAt}`,
    '',
    '## Headline',
    '',
    `- Cases run: ${report.headline.caseCount}`,
    `- Headline cases: ${report.headline.headlineCaseCount}`,
    `- Headline PK recall: ${fixed(report.headline.headlinePkRecall)}`,
    `- Headline FK recall: ${fixed(report.headline.headlineFkRecall)}`,
    `- Headline accepted-or-review recall: ${fixed(report.headline.headlineAcceptedOrReviewRecall)}`,
    `- Accepted false positives: ${report.headline.acceptedFalsePositiveCount}`,
    `- Validation-blocked cases: ${report.headline.validationBlockedCount}`,
    '',
    '## Cases',
    '',
    '| Fixture | Tier | Mode | Status | Tuning Eligible | PK Recall | FK Recall | Accepted+Review Recall | Accepted FP | Reason |',
    '| --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | --- |',
  ];

  for (const item of report.cases) {
    lines.push(
      [
        `| ${item.fixtureId}`,
        item.tier,
        item.mode,
        item.status,
        item.tuningEligible ? 'yes' : 'no',
        fixed(item.metrics.pkRecall),
        fixed(item.metrics.fkRecall),
        fixed(item.metrics.acceptedOrReviewRecall),
        String(item.metrics.acceptedFalsePositiveCount ?? '-'),
        `${item.reason ?? ''} |`,
      ].join(' | '),
    );
  }

  lines.push(...headlineFailureContextBlocks(report));
  lines.push(...failureDetailBlocks(report));
  lines.push(...compositeSkipBlocks(report));
  lines.push('');

  return `${lines.join('\n')}\n`;
}