rag-evaluation-harness

어떤 스킬인가요?

Evaluate retrieval and citation behavior for RAG pipelines from deterministic JSONL fixtures. Use when an agent needs offline Recall@K, reciprocal rank, context precision, citation coverage, citation validity, diagnostics, Markdown/JSON reports, or threshold-gated evaluation in CI.

설치 방법

프로젝트 폴더에서 아래 명령어를 실행하고, 설치할 에이전트를 선택하세요.

설치 명령을 직접 실행해야 적용됩니다. 지원 에이전트와 필요한 권한·라이선스는 제작자의 안내를 확인하세요.

스킬 원문 파일

{"id":"product-search","relevant_document_ids":["catalog-1","policy-2"],"retrieved_document_ids":["catalog-1","faq-9","policy-2"],"cited_document_ids":["catalog-1","policy-2"]}
{"id":"shipping-policy","relevant_document_ids":["policy-2"],"retrieved_document_ids":["faq-9","policy-2","policy-2"],"cited_document_ids":["policy-2"]}
{"id":"unanswerable","relevant_document_ids":[],"retrieved_document_ids":[],"cited_document_ids":[]}

#!/usr/bin/env node

import { readFile } from 'node:fs/promises'
import { pathToFileURL } from 'node:url'

const ARRAY_FIELDS = [
  'relevant_document_ids',
  'retrieved_document_ids',
  'cited_document_ids',
]

const THRESHOLD_FLAGS = {
  '--min-recall': 'recall_at_k',
  '--min-mrr': 'reciprocal_rank',
  '--min-context-precision': 'context_precision_at_k',
  '--min-citation-coverage': 'citation_coverage',
  '--min-citation-validity': 'citation_validity',
}

function unique(values) {
  return [...new Set(values)]
}

function validateCase(value, lineNumber) {
  const prefix = lineNumber ? `JSONL line ${lineNumber}` : 'case'
  if (!value || typeof value !== 'object' || Array.isArray(value)) {
    throw new Error(`${prefix} must be a JSON object`)
  }
  if (typeof value.id !== 'string' || value.id.length === 0) {
    throw new Error(`${prefix} id must be a non-empty string`)
  }
  for (const field of ARRAY_FIELDS) {
    if (!Array.isArray(value[field])) {
      throw new Error(`${prefix} ${field} must be an array`)
    }
    if (value[field].some((id) => typeof id !== 'string' || id.length === 0)) {
      throw new Error(`${prefix} ${field} must contain non-empty strings`)
    }
  }
}

export function parseCases(input) {
  const parsed = []
  for (const [index, rawLine] of input.split(/\r?\n/).entries()) {
    if (!rawLine.trim()) continue
    let value
    try {
      value = JSON.parse(rawLine)
    } catch {
      throw new Error(`JSONL line ${index + 1} is not valid JSON`)
    }
    parsed.push({ value, lineNumber: index + 1 })
  }
  const cases = []
  const ids = new Set()
  for (const { value, lineNumber } of parsed) {
    validateCase(value, lineNumber)
    if (ids.has(value.id)) {
      throw new Error(`JSONL line ${lineNumber} has duplicate case id: ${value.id}`)
    }
    ids.add(value.id)
    cases.push(value)
  }
  if (cases.length === 0) throw new Error('input contains no evaluation cases')
  return cases
}

export function evaluateCase(testCase, k) {
  validateCase(testCase)
  if (!Number.isInteger(k) || k <= 0) throw new Error('k must be a positive integer')

  const relevant = unique(testCase.relevant_document_ids)
  const retrieved = unique(testCase.retrieved_document_ids)
  const cited = unique(testCase.cited_document_ids)
  const topK = retrieved.slice(0, k)
  const relevantSet = new Set(relevant)
  const retrievedSet = new Set(retrieved)
  const citedSet = new Set(cited)
  const relevantInTopK = topK.filter((id) => relevantSet.has(id))
  const firstRelevantRank = topK.findIndex((id) => relevantSet.has(id))
  const validCitations = cited.filter((id) => retrievedSet.has(id))
  const citedRelevant = relevant.filter((id) => citedSet.has(id))

  const diagnostics = []
  if (retrieved.length === 0) diagnostics.push('empty_retrieval')
  if (relevant.length === 0) diagnostics.push('no_relevant_documents')
  if (retrieved.length !== testCase.retrieved_document_ids.length) {
    diagnostics.push('duplicate_retrieved_ids')
  }
  if (validCitations.length !== cited.length) diagnostics.push('citations_not_retrieved')
  if (cited.length === 0 || (validCitations.length === cited.length && cited.length < retrieved.length)) {
    diagnostics.push('missing_citations')
  }

  return {
    id: testCase.id,
    metrics: {
      recall_at_k: relevant.length === 0 ? null : relevantInTopK.length / relevant.length,
      reciprocal_rank: firstRelevantRank === -1 ? 0 : 1 / (firstRelevantRank + 1),
      context_precision_at_k: topK.length === 0 ? 0 : relevantInTopK.length / topK.length,
      citation_coverage: relevant.length === 0 ? null : citedRelevant.length / relevant.length,
      citation_validity: cited.length === 0 ? 0 : validCitations.length / cited.length,
    },
    diagnostics,
  }
}

function macroAverage(results, metric) {
  const values = results.map((result) => result.metrics[metric]).filter((value) => value !== null)
  return {
    value: values.length === 0 ? null : values.reduce((sum, value) => sum + value, 0) / values.length,
    count: values.length,
  }
}

export function evaluateCases(cases, k) {
  if (!Array.isArray(cases) || cases.length === 0) throw new Error('at least one evaluation case is required')
  const results = cases.map((testCase) => evaluateCase(testCase, k))
  const summary = { case_count: results.length, k }
  for (const metric of ['recall_at_k', 'reciprocal_rank', 'context_precision_at_k', 'citation_coverage', 'citation_validity']) {
    const average = macroAverage(results, metric)
    summary[metric] = average.value
    summary[`${metric}_evaluated_cases`] = average.count
  }
  const diagnostics = {}
  for (const result of results) {
    for (const diagnostic of result.diagnostics) {
      diagnostics[diagnostic] = (diagnostics[diagnostic] ?? 0) + 1
    }
  }
  return { summary, diagnostics, cases: results }
}

function formatMetric(value) {
  return value === null ? 'N/A' : value.toFixed(4)
}

export function formatMarkdown(report) {
  const labels = {
    recall_at_k: 'Recall@K',
    reciprocal_rank: 'Reciprocal rank',
    context_precision_at_k: 'Context precision@K',
    citation_coverage: 'Citation coverage',
    citation_validity: 'Citation validity',
  }
  const lines = [
    '# RAG Evaluation Report',
    '',
    `Cases: ${report.summary.case_count} | K: ${report.summary.k}`,
    '',
    '## Summary',
    '',
    '| Metric | Macro average | Evaluated cases |',
    '| --- | ---: | ---: |',
  ]
  for (const [metric, label] of Object.entries(labels)) {
    lines.push(`| ${label} | ${formatMetric(report.summary[metric])} | ${report.summary[`${metric}_evaluated_cases`]} |`)
  }
  lines.push('', '## Cases', '', '| ID | Recall@K | RR | Context precision@K | Citation coverage | Citation validity | Diagnostics |', '| --- | ---: | ---: | ---: | ---: | ---: | --- |')
  for (const result of report.cases) {
    const metrics = result.metrics
    lines.push(`| ${result.id} | ${formatMetric(metrics.recall_at_k)} | ${formatMetric(metrics.reciprocal_rank)} | ${formatMetric(metrics.context_precision_at_k)} | ${formatMetric(metrics.citation_coverage)} | ${formatMetric(metrics.citation_validity)} | ${result.diagnostics.join(', ') || 'none'} |`)
  }
  lines.push('', '## Diagnostics', '')
  const entries = Object.entries(report.diagnostics)
  if (entries.length === 0) lines.push('No diagnostics reported.')
  else for (const [name, count] of entries) lines.push(`- ${name}: ${count}`)
  return `${lines.join('\n')}\n`
}

function usage() {
  return 'Usage: node evaluate-rag.mjs <input.jsonl> --k <positive integer> [--format json|markdown] [--min-recall 0..1] [--min-mrr 0..1] [--min-context-precision 0..1] [--min-citation-coverage 0..1] [--min-citation-validity 0..1]'
}

function parseArguments(args) {
  if (args.length === 0 || args.includes('--help')) {
    if (args.includes('--help')) return { help: true }
    throw new Error('input JSONL path is required')
  }
  const options = { input: args[0], k: 5, format: 'markdown', thresholds: {} }
  if (options.input.startsWith('--')) throw new Error('input JSONL path is required')
  for (let index = 1; index < args.length; index += 2) {
    const flag = args[index]
    const value = args[index + 1]
    if (value === undefined) throw new Error(`${flag} requires a value`)
    if (flag === '--k') {
      options.k = Number(value)
      if (!Number.isInteger(options.k) || options.k <= 0) throw new Error('k must be a positive integer')
    } else if (flag === '--format') {
      if (!['json', 'markdown'].includes(value)) throw new Error('format must be json or markdown')
      options.format = value
    } else if (THRESHOLD_FLAGS[flag]) {
      const threshold = Number(value)
      if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
        throw new Error(`${flag} must be a number between 0 and 1`)
      }
      options.thresholds[THRESHOLD_FLAGS[flag]] = threshold
    } else {
      throw new Error(`unknown option: ${flag}`)
    }
  }
  return options
}

async function main() {
  let options
  try {
    options = parseArguments(process.argv.slice(2))
    if (options.help) {
      process.stdout.write(`${usage()}\n`)
      return
    }
    const cases = parseCases(await readFile(options.input, 'utf8'))
    const report = evaluateCases(cases, options.k)
    process.stdout.write(options.format === 'json' ? `${JSON.stringify(report, null, 2)}\n` : formatMarkdown(report))

    const failures = Object.entries(options.thresholds).filter(([metric, minimum]) => {
      const actual = report.summary[metric]
      return actual === null || actual < minimum
    })
    if (failures.length > 0) {
      for (const [metric, minimum] of failures) {
        process.stderr.write(`Threshold failed: ${metric}=${report.summary[metric] ?? 'N/A'} is below ${minimum}\n`)
      }
      process.exitCode = 1
    }
  } catch (error) {
    process.stderr.write(`${error.message}\n${usage()}\n`)
    process.exitCode = 2
  }
}

if (import.meta.url === pathToFileURL(process.argv[1]).href) await main()

import { describe, it } from 'node:test'
import assert from 'node:assert/strict'
import { mkdtemp, readFile, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { spawn } from 'node:child_process'
import {
  evaluateCase,
  evaluateCases,
  formatMarkdown,
  parseCases,
} from './evaluate-rag.mjs'

const healthy = {
  id: 'q1',
  relevant_document_ids: ['d1', 'd3'],
  retrieved_document_ids: ['d2', 'd1', 'd4', 'd3'],
  cited_document_ids: ['d1', 'd3'],
}

describe('RAG metrics', () => {
  it('calculates ranked retrieval and citation metrics at k', () => {
    assert.deepEqual(evaluateCase(healthy, 3), {
      id: 'q1',
      metrics: {
        recall_at_k: 0.5,
        reciprocal_rank: 0.5,
        context_precision_at_k: 1 / 3,
        citation_coverage: 1,
        citation_validity: 1,
      },
      diagnostics: ['missing_citations'],
    })
  })

  it('deduplicates retrieved IDs for metrics and reports invalid citations', () => {
    const result = evaluateCase({
      id: 'duplicates',
      relevant_document_ids: ['d1'],
      retrieved_document_ids: ['d2', 'd2', 'd1'],
      cited_document_ids: ['d1', 'd9'],
    }, 3)

    assert.equal(result.metrics.recall_at_k, 1)
    assert.equal(result.metrics.context_precision_at_k, 0.5)
    assert.equal(result.metrics.citation_coverage, 1)
    assert.equal(result.metrics.citation_validity, 0.5)
    assert.deepEqual(result.diagnostics, ['duplicate_retrieved_ids', 'citations_not_retrieved'])
  })

  it('uses null denominators for cases without relevant documents', () => {
    const result = evaluateCase({
      id: 'no-relevant',
      relevant_document_ids: [],
      retrieved_document_ids: [],
      cited_document_ids: [],
    }, 3)

    assert.deepEqual(result.metrics, {
      recall_at_k: null,
      reciprocal_rank: 0,
      context_precision_at_k: 0,
      citation_coverage: null,
      citation_validity: 0,
    })
    assert.deepEqual(result.diagnostics, ['empty_retrieval', 'no_relevant_documents', 'missing_citations'])
  })

  it('aggregates macro metrics over non-null denominators and counts diagnostics', () => {
    const report = evaluateCases([healthy, {
      id: 'q2',
      relevant_document_ids: ['d9'],
      retrieved_document_ids: ['d9'],
      cited_document_ids: ['d9'],
    }], 3)

    assert.equal(report.summary.case_count, 2)
    assert.equal(report.summary.recall_at_k, 0.75)
    assert.equal(report.summary.recall_at_k_evaluated_cases, 2)
    assert.equal(report.summary.reciprocal_rank, 0.75)
    assert.equal(report.diagnostics.missing_citations, 1)
  })
})

describe('RAG input and output', () => {
  it('parses JSONL, tolerates blank lines, and reports line-specific errors', () => {
    assert.deepEqual(parseCases('{"id":"q1","relevant_document_ids":[],"retrieved_document_ids":[],"cited_document_ids":[]}\n\n'), [{
      id: 'q1', relevant_document_ids: [], retrieved_document_ids: [], cited_document_ids: [],
    }])
    assert.throws(() => parseCases('{"id":"bad"}\nnot-json\n'), /line 2.*valid JSON/i)
    assert.throws(() => parseCases('{"id":"bad","relevant_document_ids":"d1","retrieved_document_ids":[],"cited_document_ids":[]}'), /relevant_document_ids.*array/i)
    assert.throws(() => parseCases('{"id":"q1","relevant_document_ids":[],"retrieved_document_ids":[],"cited_document_ids":[]}\n{"id":"q1","relevant_document_ids":[],"retrieved_document_ids":[],"cited_document_ids":[]}'), /duplicate case id/i)
  })

  it('formats a readable Markdown report', () => {
    const markdown = formatMarkdown(evaluateCases([healthy], 3))
    assert.match(markdown, /^# RAG Evaluation Report/m)
    assert.match(markdown, /Recall@K/)
    assert.match(markdown, /q1/)
    assert.match(markdown, /missing_citations/)
  })
})

function runCli(args) {
  return new Promise((resolve) => {
    const child = spawn(process.execPath, ['./evaluate-rag.mjs', ...args], { cwd: new URL('.', import.meta.url) })
    let stdout = ''
    let stderr = ''
    child.stdout.on('data', (chunk) => { stdout += chunk })
    child.stderr.on('data', (chunk) => { stderr += chunk })
    child.on('close', (code) => resolve({ code, stdout, stderr }))
  })
}

describe('RAG CLI', () => {
  it('emits JSON and uses exit code 1 when a threshold is not met', async () => {
    const directory = await mkdtemp(join(tmpdir(), 'rag-eval-'))
    const input = join(directory, 'cases.jsonl')
    await writeFile(input, `${JSON.stringify(healthy)}\n`)

    const result = await runCli([input, '--k', '3', '--format', 'json', '--min-recall', '1'])
    assert.equal(result.code, 1)
    const report = JSON.parse(result.stdout)
    assert.equal(report.summary.case_count, 1)
    assert.match(result.stderr, /threshold/i)
    assert.equal(await readFile(input, 'utf8'), `${JSON.stringify(healthy)}\n`)
  })

  it('rejects invalid k with exit code 2', async () => {
    const directory = await mkdtemp(join(tmpdir(), 'rag-eval-'))
    const input = join(directory, 'cases.jsonl')
    await writeFile(input, `${JSON.stringify(healthy)}\n`)

    const result = await runCli([input, '--k', '0'])
    assert.equal(result.code, 2)
    assert.match(result.stderr, /k.*positive integer/i)
  })
})

---
name: rag-evaluation-harness
category: ai-ml
description: "Evaluate retrieval and citation behavior for RAG pipelines from deterministic JSONL fixtures. Use when an agent needs offline Recall@K, reciprocal rank, context precision, citation coverage, citation validity, diagnostics, Markdown/JSON reports, or threshold-gated evaluation in CI."
---

# RAG Evaluation Harness

Use this skill to measure a retrieval-and-citation contract without making model calls or network requests. The bundled evaluator compares explicit document IDs, so it is suitable for repeatable local checks and CI gates.

## Input Contract

Provide one JSON object per line with a unique string `id` and three arrays of document IDs:

```json
{"id":"question-1","relevant_document_ids":["doc-a"],"retrieved_document_ids":["doc-b","doc-a"],"cited_document_ids":["doc-a"]}
```

Blank lines are ignored. Invalid JSON, missing arrays, non-string IDs, and duplicate case IDs fail with the JSONL line number. Keep the fixture's relevance labels and citation IDs explicit; do not infer them from answer text.

## Run an Evaluation

Set the installed skill directory and run the standard-library-only evaluator:

```bash
SKILL_DIR="<absolute path to the installed rag-evaluation-harness skill>"
node "$SKILL_DIR/scripts/evaluate-rag.mjs" "$SKILL_DIR/examples/sample-evaluation.jsonl" \\
  --k 3 --format markdown
```

Use `--format json` for CI or downstream tooling. Add any of these optional thresholds (each must be between 0 and 1):

```text
--min-recall
--min-mrr
--min-context-precision
--min-citation-coverage
--min-citation-validity
```

The process exits `0` when all requested thresholds pass, `1` when a threshold fails, and `2` for invalid arguments or input. Threshold failures are written to stderr while the complete report remains on stdout.

## Interpret the Report

- `Recall@K`: relevant IDs found in the first K unique retrieved IDs divided by all relevant IDs.
- `Reciprocal rank`: inverse rank of the first relevant retrieved ID, or zero when none is found.
- `Context precision@K`: relevant IDs in the first K unique retrieved IDs divided by the number of retrieved IDs considered.
- `Citation coverage`: relevant IDs cited divided by all relevant IDs.
- `Citation validity`: cited IDs that were retrieved divided by all cited IDs.

The summary is a macro average. Recall and citation coverage are `null` for cases with no relevant IDs and are excluded from their macro denominators. Other empty denominators are reported as zero. Diagnostics call out empty retrieval, absent relevance labels, duplicate retrieved IDs, citations that were not retrieved, and missing citations.

These are ID-level proxy metrics. They do not establish semantic answer quality, entailment, attribution correctness, or groundedness. Pair them with a separate answer-quality evaluation when those properties matter.

## Verification

Run the focused tests and the repository skill validator:

```bash
node --test "$SKILL_DIR/scripts/evaluate-rag.test.mjs"
node scripts/validate-skills.js
```

The evaluator is deterministic and offline. It reads only the supplied JSONL file and never executes retrieved content, calls an MCP server, accesses credentials, or mutates the input.