my-evals.js
Copy
Ask AI
import { createApp } from 'claudeye';
const app = createApp();
// ── Evals (actions can read these results) ───────────────────────
app.eval('under-50-turns', ({ stats }) => ({
pass: stats.turnCount <= 50,
score: Math.max(0, 1 - stats.turnCount / 100),
message: `${stats.turnCount} turn(s)`,
}));
app.eval('has-completion', ({ entries }) => {
const last = [...entries].reverse().find(e => e.type === 'assistant');
const hasText = last?.message?.content?.some?.(b => b.type === 'text');
return {
pass: !!hasText,
score: hasText ? 1.0 : 0,
message: hasText ? 'Ended with text' : 'No final text response',
};
});
// ── Enrichments ──────────────────────────────────────────────────
app.enrich('overview', ({ stats }) => ({
'Turns': stats.turnCount,
'Tool Calls': stats.toolCallCount,
'Models': stats.models.join(', ') || 'none',
}));
// ── Actions ──────────────────────────────────────────────────────
// Session summary: combines stats with eval pass counts
app.action('session-summary', ({ stats, evalResults }) => {
const evalNames = Object.keys(evalResults);
const passCount = evalNames.filter(n => evalResults[n]?.pass).length;
return {
output: [
`Session: ${stats.turnCount} turns, ${stats.toolCallCount} tool calls`,
`Duration: ${stats.duration}`,
`Models: ${stats.models.join(', ') || 'unknown'}`,
`Evals: ${passCount}/${evalNames.length} passed`,
].join('\n'),
status: 'success',
message: 'Summary generated',
};
});
// Tool inventory: lists unique tools used
app.action('tool-inventory', ({ entries }) => {
const toolNames = [...new Set(
entries
.filter(e => e.type === 'assistant' && Array.isArray(e.message?.content))
.flatMap(e => e.message.content.filter(b => b.type === 'tool_use').map(b => b.name))
)];
return {
output: toolNames.length > 0
? `Tools used:\n${toolNames.map(t => ` - ${t}`).join('\n')}`
: 'No tools used in this session',
status: 'success',
};
}, { condition: ({ stats }) => stats.toolCallCount > 0 });
// Side-effect action: write a report file (cache: false = always re-runs)
app.action('write-report', async ({ projectName, sessionId, stats }) => {
const fs = await import('fs/promises');
await fs.appendFile('session-reports.jsonl', JSON.stringify({
projectName,
sessionId,
turns: stats.turnCount,
timestamp: new Date().toISOString(),
}) + '\n');
return { status: 'success', message: 'Report appended to session-reports.jsonl' };
}, { cache: false });
app.listen();
Copy
Ask AI
claudeye --evals ./my-evals.js

