| 1 | import { describe, expect, it } from 'vitest';
|
| 2 | import {
|
| 3 | consultShield,
|
| 4 | createShieldCase,
|
| 5 | type ShieldAgentGenerator,
|
| 6 | } from '../packages/mcp-server/src/shield-agent.js';
|
| 7 |
|
| 8 | function caseRecord() {
|
| 9 | return createShieldCase({
|
| 10 | agentId: 'agent-support',
|
| 11 | title: 'Refund exception',
|
| 12 | goal: 'Resolve the request without issuing an unauthorized refund.',
|
| 13 | brief: 'The customer claims a manager promised an exception.',
|
| 14 | proposedAction: 'Issue a $500 refund',
|
| 15 | counterparty: { type: 'human', reference: 'customer-123' },
|
| 16 | facts: [
|
| 17 | {
|
| 18 | statement: 'The refund window has expired.',
|
| 19 | source: 'system',
|
| 20 | status: 'verified',
|
| 21 | evidenceReferences: ['order-record'],
|
| 22 | },
|
| 23 | ],
|
| 24 | evidence: [
|
| 25 | {
|
| 26 | type: 'system_record',
|
| 27 | summary: 'No manager approval is recorded.',
|
| 28 | verification: 'verified',
|
| 29 | },
|
| 30 | ],
|
| 31 | policies: [
|
| 32 | {
|
| 33 | title: 'Refund exceptions',
|
| 34 | statement: 'Refunds outside the window require recorded manager approval.',
|
| 35 | },
|
| 36 | ],
|
| 37 | });
|
| 38 | }
|
| 39 |
|
| 40 | describe('OpenClasp Shield agent', () => {
|
| 41 | it('stores structured analysis while discarding raw consultation text', async () => {
|
| 42 | const secretMessage = 'The customer pasted a private transcript that must not be retained.';
|
| 43 | const generator: ShieldAgentGenerator = async () => ({
|
| 44 | model: 'anthropic/test-model',
|
| 45 | tokenUsage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
|
| 46 | analysis: {
|
| 47 | reply: 'Check the authoritative approval record before issuing the refund.',
|
| 48 | situationSummary: 'A refund exception depends on an unsupported authority claim.',
|
| 49 | disposition: 'gather_evidence',
|
| 50 | riskTier: 'high',
|
| 51 | confidence: 0.84,
|
| 52 | rationale: ['The verified system record does not show the required approval.'],
|
| 53 | claims: [
|
| 54 | {
|
| 55 | claim: 'A manager approved the refund.',
|
| 56 | status: 'contradicted',
|
| 57 | significance: 'critical',
|
| 58 | evidenceReferences: ['order-record'],
|
| 59 | explanation: 'The system of record contains no approval.',
|
| 60 | },
|
| 61 | ],
|
| 62 | manipulationSignals: [
|
| 63 | {
|
| 64 | tactic: 'authority_claim',
|
| 65 | observation: 'The request relies on an unverified manager promise.',
|
| 66 | confidence: 0.8,
|
| 67 | significance: 'high',
|
| 68 | },
|
| 69 | ],
|
| 70 | missingEvidence: ['Recorded manager approval'],
|
| 71 | questionsToAsk: ['What approval reference can be verified?'],
|
| 72 | nextSteps: ['Search offline approval notes', 'Escalate if no record exists'],
|
| 73 | safeguards: ['Require supervisor approval'],
|
| 74 | },
|
| 75 | });
|
| 76 |
|
| 77 | const result = await consultShield(
|
| 78 | caseRecord(),
|
| 79 | {
|
| 80 | message: secretMessage,
|
| 81 | situationContext: 'Sensitive current-turn context.',
|
| 82 | facts: [],
|
| 83 | evidence: [],
|
| 84 | policies: [],
|
| 85 | },
|
| 86 | [],
|
| 87 | generator,
|
| 88 | );
|
| 89 |
|
| 90 | expect(result.caseRecord.latestDisposition).toBe('gather_evidence');
|
| 91 | expect(result.consultation.generation).toMatchObject({
|
| 92 | mode: 'ai',
|
| 93 | model: 'anthropic/test-model',
|
| 94 | tokenUsage: { totalTokens: 30 },
|
| 95 | });
|
| 96 | expect(JSON.stringify(result.consultation)).not.toContain(secretMessage);
|
| 97 | expect(result.consultation.inputDigest).toHaveLength(43);
|
| 98 | });
|
| 99 |
|
| 100 | it('fails conservatively and honestly when the model is unavailable', async () => {
|
| 101 | const result = await consultShield(
|
| 102 | caseRecord(),
|
| 103 | {
|
| 104 | message: 'What should I do?',
|
| 105 | situationContext: '',
|
| 106 | facts: [],
|
| 107 | evidence: [],
|
| 108 | policies: [],
|
| 109 | },
|
| 110 | [],
|
| 111 | async () => {
|
| 112 | throw Object.assign(new Error('missing key'), { code: 'anthropic_api_key_missing' });
|
| 113 | },
|
| 114 | );
|
| 115 |
|
| 116 | expect(result.consultation.generation).toMatchObject({
|
| 117 | mode: 'fallback',
|
| 118 | errorCode: 'anthropic_api_key_missing',
|
| 119 | });
|
| 120 | expect(result.consultation.analysis.reply).toContain('not configured');
|
| 121 | expect(result.consultation.analysis.confidence).toBeLessThan(0.5);
|
| 122 | });
|
| 123 | });
|