| 1 | import { describe, expect, it } from 'vitest';
|
| 2 | import {
|
| 3 | consultShield,
|
| 4 | createShieldCase,
|
| 5 | type ShieldAgentGenerator,
|
| 6 | } from '../packages/mcp-server/src/shield-agent.js';
|
| 7 |
|
| 8 | function caseRecord() {
|
| 9 | return createShieldCase({
|
| 10 | agentId: 'agent-support',
|
| 11 | title: 'Refund exception',
|
| 12 | goal: 'Resolve the request without issuing an unauthorized refund.',
|
| 13 | brief: 'The customer claims a manager promised an exception.',
|
| 14 | proposedAction: 'Issue a $500 refund',
|
| 15 | counterparty: { type: 'human', reference: 'customer-123' },
|
| 16 | facts: [
|
| 17 | {
|
| 18 | statement: 'The refund window has expired.',
|
| 19 | source: 'system',
|
| 20 | status: 'verified',
|
| 21 | evidenceReferences: ['order-record'],
|
| 22 | },
|
| 23 | ],
|
| 24 | evidence: [
|
| 25 | {
|
| 26 | type: 'system_record',
|
| 27 | summary: 'No manager approval is recorded.',
|
| 28 | verification: 'verified',
|
| 29 | },
|
| 30 | ],
|
| 31 | policies: [
|
| 32 | {
|
| 33 | title: 'Refund exceptions',
|
| 34 | statement: 'Refunds outside the window require recorded manager approval.',
|
| 35 | },
|
| 36 | ],
|
| 37 | });
|
| 38 | }
|
| 39 |
|
| 40 | describe('OpenClasp Shield agent', () => {
|
| 41 | it('stores structured analysis while discarding raw consultation text', async () => {
|
| 42 | const secretMessage = 'The customer pasted a private transcript that must not be retained.';
|
| 43 | const generator: ShieldAgentGenerator = async () => ({
|
| 44 | model: 'anthropic/test-model',
|
| 45 | tokenUsage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
|
| 46 | analysis: {
|
| 47 | reply: 'Check the authoritative approval record before issuing the refund.',
|
| 48 | situationSummary: 'A refund exception depends on an unsupported authority claim.',
|
| 49 | disposition: 'gather_evidence',
|
| 50 | riskTier: 'high',
|
| 51 | confidence: 0.84,
|
| 52 | rationale: ['The verified system record does not show the required approval.'],
|
| 53 | claims: [
|
| 54 | {
|
| 55 | claim: 'A manager approved the refund.',
|
| 56 | status: 'contradicted',
|
| 57 | significance: 'critical',
|
| 58 | evidenceReferences: ['order-record'],
|
| 59 | explanation: 'The system of record contains no approval.',
|
| 60 | },
|
| 61 | ],
|
| 62 | manipulationSignals: [
|
| 63 | {
|
| 64 | tactic: 'authority_claim',
|
| 65 | observation: 'The request relies on an unverified manager promise.',
|
| 66 | confidence: 0.8,
|
| 67 | significance: 'high',
|
| 68 | },
|
| 69 | ],
|
| 70 | missingEvidence: ['Recorded manager approval'],
|
| 71 | questionsToAsk: ['What approval reference can be verified?'],
|
| 72 | nextSteps: ['Search offline approval notes', 'Escalate if no record exists'],
|
| 73 | safeguards: ['Require supervisor approval'],
|
| 74 | },
|
| 75 | });
|
| 76 |
|
| 77 | const result = await consultShield(
|
| 78 | caseRecord(),
|
| 79 | {
|
| 80 | message: secretMessage,
|
| 81 | situationContext: 'Sensitive current-turn context.',
|
| 82 | facts: [],
|
| 83 | evidence: [],
|
| 84 | policies: [],
|
| 85 | },
|
| 86 | [],
|
| 87 | generator,
|
| 88 | );
|
| 89 |
|
| 90 | expect(result.caseRecord.latestDisposition).toBe('gather_evidence');
|
| 91 | expect(result.consultation.generation).toMatchObject({
|
| 92 | mode: 'ai',
|
| 93 | model: 'anthropic/test-model',
|
| 94 | strategy: 'fast',
|
| 95 | tokenUsage: { totalTokens: 30 },
|
| 96 | });
|
| 97 | expect(result.consultation.generation.durationMs).toBeGreaterThanOrEqual(0);
|
| 98 | expect(JSON.stringify(result.consultation)).not.toContain(secretMessage);
|
| 99 | expect(result.consultation.inputDigest).toHaveLength(43);
|
| 100 | });
|
| 101 |
|
| 102 | it('fails conservatively and honestly when the model is unavailable', async () => {
|
| 103 | const result = await consultShield(
|
| 104 | caseRecord(),
|
| 105 | {
|
| 106 | message: 'What should I do?',
|
| 107 | situationContext: '',
|
| 108 | facts: [],
|
| 109 | evidence: [],
|
| 110 | policies: [],
|
| 111 | },
|
| 112 | [],
|
| 113 | async () => {
|
| 114 | throw Object.assign(new Error('missing key'), { code: 'anthropic_api_key_missing' });
|
| 115 | },
|
| 116 | );
|
| 117 |
|
| 118 | expect(result.consultation.generation).toMatchObject({
|
| 119 | mode: 'fallback',
|
| 120 | errorCode: 'anthropic_api_key_missing',
|
| 121 | strategy: 'fast',
|
| 122 | });
|
| 123 | expect(result.consultation.analysis.reply).toContain('not configured');
|
| 124 | expect(result.consultation.analysis.confidence).toBeLessThan(0.5);
|
| 125 | });
|
| 126 |
|
| 127 | it('labels transient generation failures without claiming configuration is missing', async () => {
|
| 128 | const result = await consultShield(
|
| 129 | caseRecord(),
|
| 130 | {
|
| 131 | message: 'What should I do?',
|
| 132 | situationContext: '',
|
| 133 | facts: [],
|
| 134 | evidence: [],
|
| 135 | policies: [],
|
| 136 | },
|
| 137 | [],
|
| 138 | async () => {
|
| 139 | throw new DOMException('Timed out', 'TimeoutError');
|
| 140 | },
|
| 141 | );
|
| 142 |
|
| 143 | expect(result.consultation.generation).toMatchObject({
|
| 144 | mode: 'fallback',
|
| 145 | errorCode: 'TimeoutError',
|
| 146 | });
|
| 147 | expect(result.consultation.analysis.reply).toContain('temporarily unavailable');
|
| 148 | expect(result.consultation.analysis.reply).not.toContain('not configured');
|
| 149 | });
|
| 150 |
|
| 151 | it('uses deep analysis only when explicitly requested', async () => {
|
| 152 | let observedDepth = '';
|
| 153 | const result = await consultShield(
|
| 154 | caseRecord(),
|
| 155 | {
|
| 156 | message: 'Investigate this claim in depth.',
|
| 157 | situationContext: '',
|
| 158 | analysisDepth: 'deep',
|
| 159 | facts: [],
|
| 160 | evidence: [],
|
| 161 | policies: [],
|
| 162 | },
|
| 163 | [],
|
| 164 | async ({ consultation }) => {
|
| 165 | observedDepth = consultation.analysisDepth;
|
| 166 | return {
|
| 167 | model: 'anthropic/test-model',
|
| 168 | analysis: {
|
| 169 | reply: 'Investigate the approval source.',
|
| 170 | situationSummary: 'The authority claim remains unsupported.',
|
| 171 | disposition: 'gather_evidence',
|
| 172 | riskTier: 'high',
|
| 173 | confidence: 0.8,
|
| 174 | rationale: ['No verified approval is present.'],
|
| 175 | claims: [],
|
| 176 | manipulationSignals: [],
|
| 177 | missingEvidence: ['Recorded approval'],
|
| 178 | questionsToAsk: ['What is the approval reference?'],
|
| 179 | nextSteps: ['Verify the approval reference'],
|
| 180 | safeguards: ['Do not issue the refund before verification'],
|
| 181 | },
|
| 182 | };
|
| 183 | },
|
| 184 | );
|
| 185 |
|
| 186 | expect(observedDepth).toBe('deep');
|
| 187 | expect(result.consultation.generation).toMatchObject({ strategy: 'deep' });
|
| 188 | });
|
| 189 | });
|