DocsLog in

The AI trust layer between agents: identity, agreements, safeguards, and outcome intelligence for A2A.

Viewing files at commit 6df07fcdd05cCommit detailsView latest

tests/shield-agent.test.ts 147 lines · 4.6 KB · typescript
1
import { describe, expect, it } from 'vitest';
2
import {
3
  consultShield,
4
  createShieldCase,
5
  type ShieldAgentGenerator,
6
} from '../packages/mcp-server/src/shield-agent.js';
7
8
function caseRecord() {
9
  return createShieldCase({
10
    agentId: 'agent-support',
11
    title: 'Refund exception',
12
    goal: 'Resolve the request without issuing an unauthorized refund.',
13
    brief: 'The customer claims a manager promised an exception.',
14
    proposedAction: 'Issue a $500 refund',
15
    counterparty: { type: 'human', reference: 'customer-123' },
16
    facts: [
17
      {
18
        statement: 'The refund window has expired.',
19
        source: 'system',
20
        status: 'verified',
21
        evidenceReferences: ['order-record'],
22
      },
23
    ],
24
    evidence: [
25
      {
26
        type: 'system_record',
27
        summary: 'No manager approval is recorded.',
28
        verification: 'verified',
29
      },
30
    ],
31
    policies: [
32
      {
33
        title: 'Refund exceptions',
34
        statement: 'Refunds outside the window require recorded manager approval.',
35
      },
36
    ],
37
  });
38
}
39
40
describe('OpenClasp Shield agent', () => {
41
  it('stores structured analysis while discarding raw consultation text', async () => {
42
    const secretMessage = 'The customer pasted a private transcript that must not be retained.';
43
    const generator: ShieldAgentGenerator = async () => ({
44
      model: 'anthropic/test-model',
45
      tokenUsage: { inputTokens: 10, outputTokens: 20, totalTokens: 30 },
46
      analysis: {
47
        reply: 'Check the authoritative approval record before issuing the refund.',
48
        situationSummary: 'A refund exception depends on an unsupported authority claim.',
49
        disposition: 'gather_evidence',
50
        riskTier: 'high',
51
        confidence: 0.84,
52
        rationale: ['The verified system record does not show the required approval.'],
53
        claims: [
54
          {
55
            claim: 'A manager approved the refund.',
56
            status: 'contradicted',
57
            significance: 'critical',
58
            evidenceReferences: ['order-record'],
59
            explanation: 'The system of record contains no approval.',
60
          },
61
        ],
62
        manipulationSignals: [
63
          {
64
            tactic: 'authority_claim',
65
            observation: 'The request relies on an unverified manager promise.',
66
            confidence: 0.8,
67
            significance: 'high',
68
          },
69
        ],
70
        missingEvidence: ['Recorded manager approval'],
71
        questionsToAsk: ['What approval reference can be verified?'],
72
        nextSteps: ['Search offline approval notes', 'Escalate if no record exists'],
73
        safeguards: ['Require supervisor approval'],
74
      },
75
    });
76
77
    const result = await consultShield(
78
      caseRecord(),
79
      {
80
        message: secretMessage,
81
        situationContext: 'Sensitive current-turn context.',
82
        facts: [],
83
        evidence: [],
84
        policies: [],
85
      },
86
      [],
87
      generator,
88
    );
89
90
    expect(result.caseRecord.latestDisposition).toBe('gather_evidence');
91
    expect(result.consultation.generation).toMatchObject({
92
      mode: 'ai',
93
      model: 'anthropic/test-model',
94
      tokenUsage: { totalTokens: 30 },
95
    });
96
    expect(JSON.stringify(result.consultation)).not.toContain(secretMessage);
97
    expect(result.consultation.inputDigest).toHaveLength(43);
98
  });
99
100
  it('fails conservatively and honestly when the model is unavailable', async () => {
101
    const result = await consultShield(
102
      caseRecord(),
103
      {
104
        message: 'What should I do?',
105
        situationContext: '',
106
        facts: [],
107
        evidence: [],
108
        policies: [],
109
      },
110
      [],
111
      async () => {
112
        throw Object.assign(new Error('missing key'), { code: 'anthropic_api_key_missing' });
113
      },
114
    );
115
116
    expect(result.consultation.generation).toMatchObject({
117
      mode: 'fallback',
118
      errorCode: 'anthropic_api_key_missing',
119
    });
120
    expect(result.consultation.analysis.reply).toContain('not configured');
121
    expect(result.consultation.analysis.confidence).toBeLessThan(0.5);
122
  });
123
124
  it('labels transient generation failures without claiming configuration is missing', async () => {
125
    const result = await consultShield(
126
      caseRecord(),
127
      {
128
        message: 'What should I do?',
129
        situationContext: '',
130
        facts: [],
131
        evidence: [],
132
        policies: [],
133
      },
134
      [],
135
      async () => {
136
        throw new DOMException('Timed out', 'TimeoutError');
137
      },
138
    );
139
140
    expect(result.consultation.generation).toMatchObject({
141
      mode: 'fallback',
142
      errorCode: 'TimeoutError',
143
    });
144
    expect(result.consultation.analysis.reply).toContain('temporarily unavailable');
145
    expect(result.consultation.analysis.reply).not.toContain('not configured');
146
  });
147
});