MCP Testing Framework

npx sunpeak test init --server http://localhost:8000/mcp
npx sunpeak test

npx sunpeak test init --server "python my_server.py"

import { test, expect } from 'sunpeak/test';

test('should render album cards in light mode', async ({ inspector }) => {
  const result = await inspector.renderTool('show-albums', {}, { theme: 'light' });
  const app = result.app();
  await expect(app.locator('button:has-text("Summer Slice")')).toBeVisible();
});

test('should render in fullscreen mode', async ({ inspector }) => {
  const result = await inspector.renderTool('show-albums', {}, { displayMode: 'fullscreen' });
  const app = result.app();
  await expect(app.locator('button:has-text("Summer Slice")')).toBeVisible();
});

// playwright.config.ts
import { defineConfig } from 'sunpeak/test/config';
export default defineConfig();

// tests/simulations/review-purchase.json
{
  "tool": "review-purchase",
  "toolResult": { "structuredContent": { "..." } },
  "serverTools": {
    "review": [
      {
        "when": { "confirmed": true },
        "result": {
          "content": [{ "type": "text", "text": "Completed." }],
          "structuredContent": { "status": "success", "message": "Completed." }
        }
      },
      {
        "when": { "confirmed": false },
        "result": {
          "content": [{ "type": "text", "text": "Cancelled." }],
          "structuredContent": { "status": "cancelled", "message": "Cancelled." }
        }
      }
    ]
  }
}

import { test, expect } from 'sunpeak/test';

test('should show success when server confirms', async ({ inspector }) => {
  const result = await inspector.renderTool('review-purchase');
  const app = result.app();

  await app.locator('button:has-text("Place Order")').evaluate((el) => (el as HTMLElement).click());
  await expect(app.locator('text=Completed.')).toBeVisible({ timeout: 10000 });
});

import { test, expect } from 'sunpeak/test';

test('should render album cards', async ({ inspector }) => {
  const result = await inspector.renderTool('show-albums', {}, { theme: 'light' });
  const app = result.app();
  await expect(app.locator('button:has-text("Summer Slice")')).toBeVisible();
});

test('should render with dark theme', async ({ inspector }) => {
  const result = await inspector.renderTool('show-albums', {}, { theme: 'dark' });
  const app = result.app();
  await expect(app.locator('button:has-text("Summer Slice")')).toBeVisible();
});

test('should render in fullscreen', async ({ inspector }) => {
  const result = await inspector.renderTool('show-albums', {}, { displayMode: 'fullscreen' });
  const app = result.app();
  await expect(app.locator('button:has-text("Summer Slice")')).toBeVisible();
});

import { test, expect } from 'sunpeak/test';

test('albums renders correctly', async ({ inspector }) => {
  const result = await inspector.renderTool('show-albums', {}, { theme: 'light' });
  const app = result.app();
  await expect(app.locator('button:has-text("Summer Slice")')).toBeVisible();

  await result.screenshot('albums-light');
});

await result.screenshot('card', { element: app.locator('.card') });

import { defineConfig } from 'sunpeak/test/config';
export default defineConfig({
  visual: {
    threshold: 0.2,
    maxDiffPixelRatio: 0.05,
  },
});

// tests/live/weather.spec.ts
import { test, expect } from 'sunpeak/test/live';

test('weather tool renders forecast', async ({ live }) => {
  // invoke() starts a new chat, sends the prompt, and returns the app iframe
  const app = await live.invoke('show me the weather in Austin');
  await expect(app.locator('h1')).toBeVisible();
});

// tests/live/playwright.config.ts
import { defineLiveConfig } from 'sunpeak/test/live/config';
export default defineLiveConfig();

import { defineLiveConfig } from 'sunpeak/test/live/config';
export default defineLiveConfig({ hosts: ['chatgpt', 'claude'] });

import { defineEvalConfig } from 'sunpeak/eval';

// API keys are loaded automatically from tests/evals/.env (gitignored).

export default defineEvalConfig({
  // Server is auto-detected for sunpeak projects.
  // For non-sunpeak projects: server: 'http://localhost:8000/mcp',

  models: ['gpt-4o', 'o4-mini', 'claude-sonnet-4-20250514', 'gemini-2.0-flash'],

  defaults: {
    runs: 10,          // Run each case 10 times per model
    maxSteps: 1,       // Max tool call steps per run
    temperature: 0,    // Most deterministic results
    timeout: 30_000,   // Timeout per run in ms
  },
});

# tests/evals/.env
OPENAI_API_KEY=sk-...
ANTHROPIC_API_KEY=sk-ant-...
GOOGLE_GENERATIVE_AI_API_KEY=...

// tests/evals/show-albums.eval.ts
import { expect } from 'vitest';
import { defineEval } from 'sunpeak/eval';

export default defineEval({
  cases: [
    {
      name: 'food category request',
      prompt: 'Show me photos from my Austin pizza tour',
      expect: {
        tool: 'show-albums',
        args: { category: expect.stringContaining('food') },
      },
    },
    {
      name: 'multi-step review flow',
      prompt: 'Write a launch post for X and LinkedIn',
      maxSteps: 3,
      expect: [
        { tool: 'review-post' },
        { tool: 'publish-post' },
      ],
    },
    {
      name: 'follow-up uses selected app state',
      prompt: 'Book this one',
      appContext: {
        structuredContent: {
          selectedFlight: { carrier: 'delta', flightNumber: 'DL123' },
        },
      },
      expect: {
        tool: 'book-flight',
        args: { carrier: 'delta' },
      },
    },
  ],
});

show-albums
  food category request
    ✓ gpt-4o           10/10 passed (100%)  avg 1200ms
    ~ claude-sonnet     9/10 passed  (90%)   avg 800ms
    ✗ gemini-flash      6/10 passed  (60%)   avg 900ms
      └ called 'get-photos' instead of 'show-albums' (4x)

Summary: 25/30 passed (83%) across 3 model(s)

export default defineEval({
  models: ['gpt-4o'],        // Only test this eval against GPT-4o
  runs: 5,                    // Override run count
  threshold: 0.8,             // Pass if 80%+ of runs succeed (default: 100%)
  cases: [/* ... */],
});