LLM
Evals
Academy

// Generate synthetic eval cases from seed examples
import { z } from "zod";

const EvalCase = z.object({
  input: z.string(),
  expectedBehavior: z.string(),
  category: z.string(),
  difficulty: z.enum(["easy", "medium", "hard", "adversarial"]),
});

async function generateSyntheticCases(
  seedCases: z.infer<typeof EvalCase>[],
  categories: string[],
  countPerCategory: number,
): Promise<z.infer<typeof EvalCase>[]> {
  const allCases: z.infer<typeof EvalCase>[] = [];

  for (const category of categories) {
    const seeds = seedCases.filter((c) => c.category === category);

    const generated = await llm.generate({
      system: `You are an eval dataset generator. Given seed examples,
generate ${countPerCategory} new test cases for the "${category}" category.

Requirements:
- Each case must be meaningfully different from seeds
- Include a mix of difficulties: easy, medium, hard, adversarial
- Adversarial cases should test edge cases and failure modes
- Expected behavior should describe what a GOOD response does

Respond as a JSON array matching the schema.`,
      messages: [
        {
          role: "user",
          content: `Seed examples:\n${JSON.stringify(seeds, null, 2)}`,
        },
      ],
    });

    const parsed = z.array(EvalCase).parse(JSON.parse(generated));
    allCases.push(...parsed);
  }

  return allCases;
}

// Sample production logs for eval dataset
interface ProductionLog {
  id: string;
  input: string;
  output: string;
  timestamp: string;
  userFeedback?: "positive" | "negative";
  latencyMs: number;
  model: string;
}

async function sampleForEvals(
  logs: ProductionLog[],
  config: {
    totalSamples: number;
    failureOversampling: number; // e.g., 3x
    stratifyBy: "category" | "date" | "model";
  },
): Promise<ProductionLog[]> {
  // Separate successes and failures
  const failures = logs.filter((l) => l.userFeedback === "negative");
  const successes = logs.filter((l) => l.userFeedback !== "negative");

  // Over-sample failures (they're more valuable as eval cases)
  const failureSamples = stratifiedSample(
    failures,
    Math.min(
      failures.length,
      Math.floor(config.totalSamples * 0.4 * config.failureOversampling),
    ),
    config.stratifyBy,
  );

  const successSamples = stratifiedSample(
    successes,
    config.totalSamples - failureSamples.length,
    config.stratifyBy,
  );

  return [...failureSamples, ...successSamples];
}

// Version and track eval datasets
interface DatasetVersion {
  version: string;
  createdAt: string;
  cases: EvalCase[];
  metadata: {
    totalCases: number;
    bySource: Record<string, number>;
    byCategory: Record<string, number>;
    byDifficulty: Record<string, number>;
  };
  changelog: string;
}

async function saveDatasetVersion(
  cases: EvalCase[],
  changelog: string,
): Promise<DatasetVersion> {
  const version: DatasetVersion = {
    version: `v${Date.now()}`,
    createdAt: new Date().toISOString(),
    cases,
    metadata: {
      totalCases: cases.length,
      bySource: countBy(cases, "source"),
      byCategory: countBy(cases, "category"),
      byDifficulty: countBy(cases, "difficulty"),
    },
    changelog,
  };

  // Store in version control alongside code
  await writeFile(
    `evals/datasets/${version.version}.json`,
    JSON.stringify(version, null, 2),
  );

  // Update the "latest" symlink
  await writeFile(
    "evals/datasets/latest.json",
    JSON.stringify(version, null, 2),
  );

  return version;
}

// LLM-as-Judge with calibration and bias mitigation
import { z } from "zod";

const JudgmentSchema = z.object({
  reasoning: z.string(),
  scores: z.object({
    accuracy: z.number().min(1).max(5),
    helpfulness: z.number().min(1).max(5),
    safety: z.number().min(1).max(5),
  }),
  overall: z.number().min(1).max(5),
});

type Judgment = z.infer<typeof JudgmentSchema>;

async function llmJudge(
  response: string,
  reference: string,
  rubric: string,
  calibrationExamples: { response: string; score: number }[],
): Promise<Judgment> {
  // Include calibration examples to anchor scoring
  const calibration = calibrationExamples
    .map((ex) => `Response: "${ex.response}" → Score: ${ex.score}/5`)
    .join("\n");

  const result = await llm.generate({
    model: "gpt-4o",
    temperature: 0, // Reduce variance between runs
    system: `You are an expert evaluator. Score responses 1-5.

## Rubric
${rubric}

## Calibration Examples (use these to anchor your scale)
${calibration}

## Reference Answer
${reference}

## Instructions
1. Write step-by-step reasoning BEFORE scoring
2. Score each dimension independently
3. Overall = weighted average (accuracy 40%, helpfulness 40%, safety 20%)
4. Output valid JSON matching the schema.`,
    messages: [
      { role: "user", content: `Evaluate this response:\n"${response}"` },
    ],
  });

  return JudgmentSchema.parse(JSON.parse(result));
}

// Combined eval pipeline: assertions + similarity + judge
interface EvalResult {
  assertions: { passed: boolean; failures: string[] };
  similarity: { score: number; threshold: number };
  judge: { overall: number; reasoning: string };
  finalScore: number;
  passed: boolean;
}

async function evaluate(
  response: string,
  testCase: EvalCase,
): Promise<EvalResult> {
  // Layer 1: Fast assertions (deterministic, free)
  const assertions = runAssertions(response, testCase.assertions);

  // Short-circuit: if assertions fail, skip expensive evals
  if (!assertions.passed) {
    return {
      assertions,
      similarity: { score: 0, threshold: 0.75 },
      judge: { overall: 0, reasoning: "Skipped — assertions failed" },
      finalScore: 0,
      passed: false,
    };
  }

  // Layer 2: Semantic similarity (cheap, fast)
  const similarity = await computeSimilarity(
    response,
    testCase.referenceAnswer,
  );

  // Layer 3: LLM-as-judge (expensive, nuanced)
  const judge = await llmJudge(
    response,
    testCase.referenceAnswer,
    testCase.rubric,
    CALIBRATION_SET,
  );

  // Weighted final score
  const finalScore =
    (assertions.passed ? 0.2 : 0) +
    similarity.score * 0.3 +
    (judge.overall / 5) * 0.5;

  return {
    assertions,
    similarity,
    judge,
    finalScore,
    passed: finalScore >= testCase.threshold,
  };
}

// Annotation guidelines template
interface AnnotationGuideline {
  taskDescription: string;
  dimensions: {
    name: string;
    description: string;
    scale: { value: number; label: string; examples: string }[];
  }[];
  edgeCases: { scenario: string; guidance: string }[];
}

const summaryEvalGuideline: AnnotationGuideline = {
  taskDescription: `Evaluate the quality of an AI-generated summary.
Read the source document, then rate the summary on each dimension.`,
  dimensions: [
    {
      name: "Accuracy",
      description: "Are all facts in the summary correct per the source?",
      scale: [
        { value: 1, label: "Major errors", examples: "States wrong numbers, inverts conclusions" },
        { value: 2, label: "Minor errors", examples: "Slightly wrong dates, imprecise wording" },
        { value: 3, label: "Mostly accurate", examples: "Key facts correct, minor omissions" },
        { value: 4, label: "Accurate", examples: "All facts verifiable against source" },
        { value: 5, label: "Perfectly accurate", examples: "Every claim traceable to source" },
      ],
    },
    {
      name: "Completeness",
      description: "Does the summary cover all key points from the source?",
      scale: [
        { value: 1, label: "Missing critical info", examples: "Main conclusion absent" },
        { value: 2, label: "Significant gaps", examples: "2+ key points missing" },
        { value: 3, label: "Adequate coverage", examples: "Main points present, details missing" },
        { value: 4, label: "Good coverage", examples: "All key points, most details" },
        { value: 5, label: "Comprehensive", examples: "All points covered proportionally" },
      ],
    },
    {
      name: "Conciseness",
      description: "Is the summary appropriately brief without losing meaning?",
      scale: [
        { value: 1, label: "Far too long", examples: "Longer than source, excessive repetition" },
        { value: 2, label: "Too verbose", examples: "Could be 50% shorter without loss" },
        { value: 3, label: "Acceptable", examples: "Some unnecessary content" },
        { value: 4, label: "Concise", examples: "Tight writing, minimal fluff" },
        { value: 5, label: "Perfectly concise", examples: "Every word earns its place" },
      ],
    },
  ],
  edgeCases: [
    {
      scenario: "Summary adds information not in the source",
      guidance: "Score Accuracy as 1 regardless of whether the added info is correct.",
    },
    {
      scenario: "Summary is a single sentence",
      guidance: "Can still score high on Conciseness if it captures the core message.",
    },
  ],
};

// Calculate inter-rater reliability (Cohen's Kappa)
function cohensKappa(
  rater1: number[],
  rater2: number[],
): { kappa: number; interpretation: string } {
  const n = rater1.length;
  const categories = [...new Set([...rater1, ...rater2])];

  // Observed agreement
  let agreed = 0;
  for (let i = 0; i < n; i++) {
    if (rater1[i] === rater2[i]) agreed++;
  }
  const observedAgreement = agreed / n;

  // Expected agreement by chance
  let expectedAgreement = 0;
  for (const cat of categories) {
    const p1 = rater1.filter((r) => r === cat).length / n;
    const p2 = rater2.filter((r) => r === cat).length / n;
    expectedAgreement += p1 * p2;
  }

  const kappa = (observedAgreement - expectedAgreement) / (1 - expectedAgreement);

  const interpretation =
    kappa >= 0.81 ? "Almost perfect agreement" :
    kappa >= 0.61 ? "Substantial agreement" :
    kappa >= 0.41 ? "Moderate agreement" :
    kappa >= 0.21 ? "Fair agreement" :
    "Poor agreement — revise guidelines";

  return { kappa, interpretation };
}

// Usage: validate annotator consistency
const rater1Scores = [5, 4, 3, 5, 2, 4, 3, 5, 4, 3];
const rater2Scores = [5, 4, 4, 5, 2, 3, 3, 5, 4, 3];
const { kappa, interpretation } = cohensKappa(rater1Scores, rater2Scores);
// kappa: 0.72 — "Substantial agreement"

# promptfoo config: promptfooconfig.yaml
prompts:
  - id: current
    raw: "You are a helpful assistant. Answer: {{query}}"
  - id: candidate
    raw: |
      You are a customer support agent for Acme Corp.
      Answer the user's question accurately and concisely.
      If unsure, say so rather than guessing.
      Question: {{query}}

providers:
  - openai:gpt-4o
  - anthropic:messages:claude-sonnet-4-20250514

tests:
  - vars:
      query: "What is your refund policy?"
    assert:
      - type: contains
        value: "30 days"
      - type: llm-rubric
        value: "Response accurately describes the refund policy"
      - type: not-contains
        value: "I don't know"

  - vars:
      query: "How do I cancel my subscription?"
    assert:
      - type: contains
        value: "settings"
      - type: similar
        value: "Go to Settings > Subscription > Cancel"
        threshold: 0.7

  - vars:
      query: "Can I get a refund for a digital purchase?"
    assert:
      - type: llm-rubric
        value: "Response correctly states digital purchases are credit-only"
      - type: javascript
        value: "output.length < 500"

// Braintrust eval with custom scoring
import { Eval } from "braintrust";

Eval("customer-support-qa", {
  data: () => loadDataset("datasets/support-evals.jsonl"),

  task: async (input) => {
    const response = await llm.generate({
      system: SYSTEM_PROMPT,
      messages: [{ role: "user", content: input.query }],
    });
    return response;
  },

  scores: [
    // Built-in scorer: factual accuracy via LLM judge
    Factuality,

    // Custom scorer: check required content
    (args) => {
      const hasRequiredInfo = args.input.requiredKeywords
        .every((kw: string) => args.output.toLowerCase().includes(kw));
      return {
        name: "contains_required_info",
        score: hasRequiredInfo ? 1 : 0,
      };
    },

    // Custom scorer: response length check
    (args) => ({
      name: "conciseness",
      score: args.output.length < 500 ? 1 : args.output.length < 800 ? 0.5 : 0,
    }),
  ],
});

// Custom eval framework — minimal but effective
import { z } from "zod";

interface EvalConfig {
  name: string;
  dataset: string;
  evaluators: Evaluator[];
  thresholds: { overall: number; perCategory: Record<string, number> };
}

interface Evaluator {
  name: string;
  weight: number;
  evaluate: (input: string, output: string, expected?: string) => Promise<number>;
}

async function runEvalSuite(config: EvalConfig): Promise<EvalReport> {
  const dataset = await loadDataset(config.dataset);
  const results: EvalResult[] = [];

  for (const testCase of dataset) {
    const output = await generateResponse(testCase.input);
    const scores: Record<string, number> = {};

    for (const evaluator of config.evaluators) {
      scores[evaluator.name] = await evaluator.evaluate(
        testCase.input,
        output,
        testCase.expected,
      );
    }

    const overall = config.evaluators.reduce(
      (sum, ev) => sum + scores[ev.name] * ev.weight, 0
    );

    results.push({ testCase: testCase.id, scores, overall, output });
  }

  return generateReport(results, config.thresholds);
}

// Baseline management for regression detection
interface Baseline {
  id: string;
  promptVersion: string;
  modelVersion: string;
  timestamp: string;
  results: Map<string, CaseResult>;
  aggregates: {
    mean: number;
    median: number;
    p5: number;   // 5th percentile — worst-case performance
    p95: number;  // 95th percentile — best-case performance
    passRate: number;
  };
}

interface CaseResult {
  caseId: string;
  score: number;
  passed: boolean;
  output: string;
  evalDetails: Record<string, number>;
}

async function createBaseline(
  promptVersion: string,
  dataset: EvalCase[],
): Promise<Baseline> {
  const results = new Map<string, CaseResult>();

  for (const testCase of dataset) {
    const output = await generateResponse(testCase.input, promptVersion);
    const evalResult = await evaluate(output, testCase);

    results.set(testCase.id, {
      caseId: testCase.id,
      score: evalResult.score,
      passed: evalResult.passed,
      output,
      evalDetails: evalResult.details,
    });
  }

  const scores = [...results.values()].map((r) => r.score).sort((a, b) => a - b);

  return {
    id: `baseline-${Date.now()}`,
    promptVersion,
    modelVersion: getCurrentModelVersion(),
    timestamp: new Date().toISOString(),
    results,
    aggregates: {
      mean: average(scores),
      median: scores[Math.floor(scores.length / 2)],
      p5: scores[Math.floor(scores.length * 0.05)],
      p95: scores[Math.floor(scores.length * 0.95)],
      passRate: [...results.values()].filter((r) => r.passed).length / results.size,
    },
  };
}

// Regression detection: compare candidate against baseline
interface RegressionReport {
  improved: { caseId: string; oldScore: number; newScore: number }[];
  regressed: { caseId: string; oldScore: number; newScore: number }[];
  unchanged: string[];
  newCases: string[];  // Cases in candidate but not in baseline
  aggregateDelta: {
    mean: number;
    p5: number;
    passRate: number;
  };
  verdict: "pass" | "fail" | "review";
}

async function detectRegressions(
  baseline: Baseline,
  candidateResults: Map<string, CaseResult>,
  config: {
    regressionThreshold: number; // Score drop that counts as regression
    maxRegressions: number;      // Max allowed regressions
    minP5: number;               // Minimum acceptable p5 score
  },
): Promise<RegressionReport> {
  const report: RegressionReport = {
    improved: [],
    regressed: [],
    unchanged: [],
    newCases: [],
    aggregateDelta: { mean: 0, p5: 0, passRate: 0 },
    verdict: "pass",
  };

  for (const [caseId, candidateResult] of candidateResults) {
    const baselineResult = baseline.results.get(caseId);

    if (!baselineResult) {
      report.newCases.push(caseId);
      continue;
    }

    const delta = candidateResult.score - baselineResult.score;

    if (delta > config.regressionThreshold) {
      report.improved.push({
        caseId, oldScore: baselineResult.score, newScore: candidateResult.score,
      });
    } else if (delta < -config.regressionThreshold) {
      report.regressed.push({
        caseId, oldScore: baselineResult.score, newScore: candidateResult.score,
      });
    } else {
      report.unchanged.push(caseId);
    }
  }

  // Calculate aggregate deltas
  const candidateScores = [...candidateResults.values()].map((r) => r.score).sort((a, b) => a - b);
  const candidateP5 = candidateScores[Math.floor(candidateScores.length * 0.05)];

  report.aggregateDelta = {
    mean: average(candidateScores) - baseline.aggregates.mean,
    p5: candidateP5 - baseline.aggregates.p5,
    passRate: (candidateResults.size > 0
      ? [...candidateResults.values()].filter((r) => r.passed).length / candidateResults.size
      : 0) - baseline.aggregates.passRate,
  };

  // Determine verdict
  if (report.regressed.length > config.maxRegressions) {
    report.verdict = "fail";
  } else if (candidateP5 < config.minP5) {
    report.verdict = "fail";
  } else if (report.regressed.length > 0) {
    report.verdict = "review"; // Some regressions but within tolerance
  }

  return report;
}

// Eval-Driven Development workflow
interface EDDConfig {
  issue: string;
  evalCases: EvalCase[];
  currentPrompt: string;
  maxIterations: number;
  targetScore: number;
}

async function evalDrivenDevelopment(config: EDDConfig) {
  // Step 1: Add new eval cases to the suite
  const fullSuite = await loadEvalDataset("regression-suite");
  const expandedSuite = [...fullSuite, ...config.evalCases];

  // Step 2: Measure baseline on new cases
  const baselineResults = await runEvals(config.currentPrompt, config.evalCases);
  console.log(`Baseline: ${baselineResults.passRate * 100}% pass rate`);
  console.log(`Target: ${config.targetScore * 100}% pass rate`);

  // Step 3: Iterate with eval feedback
  let bestPrompt = config.currentPrompt;
  let bestScore = baselineResults.overallScore;
  const history: { prompt: string; score: number; delta: number }[] = [];

  for (let i = 0; i < config.maxIterations; i++) {
    // Generate a prompt variant (can be manual or LLM-assisted)
    const candidate = await generatePromptVariant(bestPrompt, {
      issue: config.issue,
      failingCases: baselineResults.failures,
      iteration: i,
    });

    // Run eval on new cases
    const candidateResults = await runEvals(candidate, config.evalCases);

    // Run full regression suite to check for side effects
    const regressionResults = await runEvals(candidate, fullSuite);

    const improved = candidateResults.overallScore > bestScore;
    const noRegressions = regressionResults.regressions.length === 0;

    history.push({
      prompt: candidate,
      score: candidateResults.overallScore,
      delta: candidateResults.overallScore - bestScore,
    });

    if (improved && noRegressions) {
      bestPrompt = candidate;
      bestScore = candidateResults.overallScore;
      console.log(`Iteration ${i + 1}: Improved to ${bestScore}`);
    }

    // Early exit if target reached
    if (bestScore >= config.targetScore) {
      console.log(`Target reached at iteration ${i + 1}`);
      break;
    }
  }

  // Step 4: Deploy if improved
  if (bestScore > baselineResults.overallScore) {
    await deploy(bestPrompt);
    // Promote new cases to permanent regression suite
    await saveEvalDataset("regression-suite", expandedSuite);
    console.log(`Deployed. Score: ${baselineResults.overallScore} → ${bestScore}`);
  }

  return { bestPrompt, bestScore, history };
}

# .github/workflows/llm-evals.yml
name: LLM Eval Suite

on:
  pull_request:
    paths:
      - 'prompts/**'
      - 'evals/**'
      - 'src/lib/llm-config.ts'

env:
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

jobs:
  fast-evals:
    name: Fast Evals (Assertions)
    runs-on: ubuntu-latest
    timeout-minutes: 2
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20

      - run: npm ci
      - run: npx tsx evals/run-assertions.ts
        env:
          EVAL_DATASET: evals/datasets/latest.json
          FAIL_THRESHOLD: 0.95

  full-evals:
    name: Full Eval Suite (LLM-as-Judge)
    needs: fast-evals
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20

      - run: npm ci

      # Run full eval suite
      - run: npx tsx evals/run-full-suite.ts
        id: evals
        env:
          EVAL_DATASET: evals/datasets/latest.json
          BASELINE_PATH: evals/baselines/latest.json
          OUTPUT_PATH: evals/results/current.json

      # Post results as PR comment
      - uses: actions/github-script@v7
        with:
          script: |
            const results = require('./evals/results/current.json');
            const body = formatEvalComment(results);
            await github.rest.issues.createComment({
              ...context.repo,
              issue_number: context.issue.number,
              body,
            });

      # Fail if below threshold
      - run: |
          node -e "
            const r = require('./evals/results/current.json');
            if (r.regressions > 0 || r.overallScore < 0.9) {
              console.error('Eval failed:', JSON.stringify(r.summary));
              process.exit(1);
            }
          "

// CI eval runner with reporting
interface CIEvalConfig {
  datasetPath: string;
  baselinePath: string;
  outputPath: string;
  thresholds: {
    fastEvalPassRate: number;    // e.g., 0.95
    fullEvalMinScore: number;    // e.g., 0.85
    maxRegressions: number;      // e.g., 3
    safetyPassRate: number;      // e.g., 1.0 (zero tolerance)
  };
}

async function runCIEvals(config: CIEvalConfig) {
  const dataset = await loadDataset(config.datasetPath);
  const baseline = await loadBaseline(config.baselinePath);

  // Partition test cases by type
  const functional = dataset.filter((c) => c.type === "functional");
  const quality = dataset.filter((c) => c.type === "quality");
  const safety = dataset.filter((c) => c.type === "safety");

  // Phase 1: Fast functional evals (parallel)
  const functionalResults = await Promise.all(
    functional.map((c) => runFunctionalEval(c))
  );
  const functionalPassRate = functionalResults.filter((r) => r.passed).length / functional.length;

  if (functionalPassRate < config.thresholds.fastEvalPassRate) {
    return { status: "fail", phase: "functional", passRate: functionalPassRate };
  }

  // Phase 2: Safety evals (zero tolerance)
  const safetyResults = await Promise.all(
    safety.map((c) => runSafetyEval(c))
  );
  const safetyPassRate = safetyResults.filter((r) => r.passed).length / safety.length;

  if (safetyPassRate < config.thresholds.safetyPassRate) {
    return { status: "fail", phase: "safety", passRate: safetyPassRate };
  }

  // Phase 3: Full quality evals with baseline comparison
  const qualityResults = await Promise.all(
    quality.map((c) => runQualityEval(c))
  );

  const regressions = detectRegressions(baseline, qualityResults);
  const overallScore = average(qualityResults.map((r) => r.score));

  // Save results for reporting
  const report = {
    status: regressions.length > config.thresholds.maxRegressions ? "fail" : "pass",
    functionalPassRate,
    safetyPassRate,
    overallScore,
    regressions: regressions.length,
    improved: qualityResults.filter((r) => r.improved).length,
    details: { functionalResults, safetyResults, qualityResults },
  };

  await saveResults(config.outputPath, report);
  return report;
}

// Format eval results as a PR comment
function formatEvalComment(results: EvalReport): string {
  const status = results.status === "pass" ? "PASSED" : "FAILED";
  const emoji = results.status === "pass" ? "white_check_mark" : "x";

  return `## LLM Eval Results: ${status} :${emoji}:

| Metric | Score | Threshold | Status |
|--------|-------|-----------|--------|
| Functional Pass Rate | ${pct(results.functionalPassRate)} | 95% | ${badge(results.functionalPassRate >= 0.95)} |
| Safety Pass Rate | ${pct(results.safetyPassRate)} | 100% | ${badge(results.safetyPassRate >= 1.0)} |
| Quality Score | ${results.overallScore.toFixed(2)}/5.0 | 4.0 | ${badge(results.overallScore >= 4.0)} |
| Regressions | ${results.regressions} | < 3 | ${badge(results.regressions < 3)} |
| Improvements | ${results.improved} | — | :chart_with_upwards_trend: |

${results.regressions > 0 ? `
### Regressed Cases
${results.details.qualityResults
  .filter((r: QualityResult) => r.regressed)
  .map((r: QualityResult) => `- **${r.caseId}**: ${r.oldScore} → ${r.newScore} (Δ ${r.delta})`)
  .join("\n")}
` : ""}

<details>
<summary>Full Results (${results.details.qualityResults.length} cases)</summary>
// ... detailed per-case results
</details>

---
*Eval suite v${results.datasetVersion} | Baseline v${results.baselineVersion}*`;
}