| 1 | """Producer-to-consumer attribution and incomplete-metric regressions."""
|
| 2 | import json
|
| 3 | import shutil
|
| 4 | import sys
|
| 5 | import tempfile
|
| 6 | import unittest
|
| 7 | from pathlib import Path
|
| 8 | from types import SimpleNamespace
|
| 9 | from unittest import mock
|
| 10 | sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
| 11 | import agent_workflow as aw
|
| 12 | import workflow_ab
|
| 13 | from oakbench import rows, agent_campaign, campaign_report
|
| 14 |
|
| 15 | class AgentOutcomeAttributionTest(unittest.TestCase):
|
| 16 | def setUp(self):
|
| 17 | self.tmp = tempfile.TemporaryDirectory()
|
| 18 | self.addCleanup(self.tmp.cleanup)
|
| 19 | self.root = Path(self.tmp.name).resolve()
|
| 20 | self.agent = aw.AgentConfig('mock', 'deterministic_mock', 'local', None, None, '1')
|
| 21 | self.subject = workflow_ab.Subject('git', 'git', 'Git', Path(shutil.which('git') or '/git'))
|
| 22 | self.workflow = workflow_ab.WORKFLOWS['bugfix_test_loop']
|
| 23 |
|
| 24 | def row(self, *, limited=False, complete=True, timeout=False, success=True, transcript=None):
|
| 25 | out, err, prompt = [self.root / name for name in ('stdout', 'stderr', 'prompt')]
|
| 26 | out.write_text(transcript if transcript is not None else 'tool: echo bounded\n'); err.write_text(''); prompt.write_text('task')
|
| 27 | execution = aw.AgentExecution(124 if timeout else 0, timeout, 1, out, err, prompt,
|
| 28 | output_limited=limited, capture_complete=complete)
|
| 29 | evidence = {'oracle_version': aw.ORACLE_VERSION, 'workflow_integrity_passed': success,
|
| 30 | 'workflow_integrity_source': 'tests_and_subject_status', 'workflow_integrity_evidence_class': 'tests_and_subject_status',
|
| 31 | 'workflow_integrity_failure_reason': None if success else 'incorrect_task', 'workflow_integrity_unmeasured_reason': None}
|
| 32 | oracle = SimpleNamespace(elapsed_ms=1, vcs_ms=1, test_ms=1, success=success, checks={'payload': success},
|
| 33 | failure_reason=None if success else 'incorrect_task', changed_files=[], test_returncode=0 if success else 1,
|
| 34 | test_output_bytes=0, dirty_files_at_end=0, commit_delta=1, bytes_written=0, artifact_paths=[], evidence=evidence)
|
| 35 | return aw.make_row('test','2026-09-18T00:00:00Z','agent-workflow',self.agent,self.subject,self.workflow,
|
| 36 | 'agent-default',0,self.root,self.root,0,0,execution,oracle,'task',{}, {}, {},'minimal','full-docs')
|
| 37 |
|
| 38 | def test_capture_fault_retains_verified_state_but_is_not_agent_failure(self):
|
| 39 | for kwargs in ({'limited': True}, {'complete': False}, {'limited': True, 'timeout': True}):
|
| 40 | row = self.row(**kwargs)
|
| 41 | self.assertIs(row['workflow_integrity_passed'], True)
|
| 42 | self.assertEqual(row['returncode'], 77)
|
| 43 | self.assertIs(row['skipped'], True)
|
| 44 | self.assertEqual(row['outcome_attribution'], 'harness')
|
| 45 | self.assertIs(row['agent_launched'], True)
|
| 46 | self.assertEqual(rows.validate_rows([row], 'agent'), [])
|
| 47 | self.assertEqual(aw.validate_row(row), [])
|
| 48 |
|
| 49 | def test_measured_success_failure_and_timeout_are_agent_attributed(self):
|
| 50 | for kwargs, expected in (({}, 'pass'), ({'success':False}, 'fail'), ({'timeout':True}, 'timeout')):
|
| 51 | row = self.row(**kwargs)
|
| 52 | self.assertEqual(row['outcome'], expected)
|
| 53 | self.assertEqual(row['outcome_attribution'], 'agent')
|
| 54 | self.assertTrue(row['agent_launched'])
|
| 55 | self.assertEqual(rows.validate_rows([row], 'agent'), [])
|
| 56 | self.assertEqual(aw.validate_row(row), [])
|
| 57 |
|
| 58 | def test_partial_transcript_has_no_token_fallback_or_summary_success_rate(self):
|
| 59 | row = self.row(limited=True)
|
| 60 | self.assertIsNone(aw.token_total(row))
|
| 61 | self.assertIsNone(rows.token_total(row))
|
| 62 | self.assertIsNone(row['token_metrics']['transcript_chars'])
|
| 63 | self.assertIsNone(row['tool_call_metrics']['tool_calls_total'])
|
| 64 | summary = aw.summary_text([row])
|
| 65 | self.assertNotIn('0.0%', summary)
|
| 66 | self.assertIn('unmeasured', summary)
|
| 67 |
|
| 68 | def test_all_partial_transcript_aggregates_are_null_but_oracle_state_survives(self):
|
| 69 | from dataclasses import replace
|
| 70 | self.agent = replace(self.agent, adapter='codex_exec_json')
|
| 71 | transcript = '\n'.join(json.dumps(event) for event in [
|
| 72 | {'type': 'item.completed', 'item': {'type': 'command_execution', 'command': 'git status', 'exit_code': 1}},
|
| 73 | {'type': 'item.completed', 'item': {'type': 'command_execution', 'command': 'python3 -m unittest', 'exit_code': 0}},
|
| 74 | {'type': 'turn.completed', 'usage': {'input_tokens': 9, 'output_tokens': 3}}])
|
| 75 | for kwargs in ({'limited': True}, {'complete': False}, {'timeout': True}):
|
| 76 | row = self.row(transcript=transcript, **kwargs)
|
| 77 | self.assertIsNone(row['tool_call_metrics']['tool_calls_by_kind'])
|
| 78 | for key, value in row['failed_command_retry_metrics'].items():
|
| 79 | if key not in {'measurement_source', 'failed_command_examples'}:
|
| 80 | self.assertIsNone(value, key)
|
| 81 | for key in ('commands_total', 'status_commands', 'diff_commands', 'snapshot_commands',
|
| 82 | 'branch_commands', 'merge_commands', 'cleanup_commands', 'branches_created'):
|
| 83 | self.assertIsNone(row['vcs_metrics'][key], key)
|
| 84 | self.assertIsNone(row['test_metrics']['commands_total'])
|
| 85 | self.assertEqual(row['vcs_metrics']['commits_created'], 1)
|
| 86 | self.assertEqual(row['vcs_metrics']['dirty_files_at_end'], 0)
|
| 87 | self.assertTrue(row['test_metrics']['required_checks_passed'])
|
| 88 | self.assertEqual(aw.validate_row(row), [])
|
| 89 |
|
| 90 | def test_setup_failure_has_own_operation_and_never_invokes_agent_or_oracle(self):
|
| 91 | fixture = workflow_ab.prepare_fixture(self.root, self.workflow)
|
| 92 | with mock.patch.object(aw, 'initialize_repo', return_value=(1,'init unavailable',5)), \
|
| 93 | mock.patch.object(aw, 'run_mock_agent') as agent, mock.patch.object(aw, 'run_oracle') as oracle:
|
| 94 | row = aw.run_one(self.agent,self.subject,self.workflow,fixture,0,self.root/'work',self.root/'artifacts',
|
| 95 | 30,'agent-default','minimal','full-docs','test','2026-09-18T00:00:00Z',{}, {})
|
| 96 | self.assertEqual(agent.call_count,0)
|
| 97 | self.assertEqual(oracle.call_count,0)
|
| 98 | self.assertEqual(row['operation'],'agent.setup')
|
| 99 | self.assertFalse(row['agent_launched'])
|
| 100 | self.assertEqual(row['setup_returncode'],5)
|
| 101 | self.assertEqual(row['returncode'],77)
|
| 102 | self.assertEqual(rows.validate_rows([row],'agent'), [])
|
| 103 | self.assertEqual(aw.validate_row(row), [])
|
| 104 |
|
| 105 | def test_actual_mock_trial_home_is_never_hashed(self):
|
| 106 | from oakbench.runlock import measurement_lock
|
| 107 | trial = self.root / "trial"
|
| 108 | fixture = workflow_ab.prepare_fixture(self.root, self.workflow)
|
| 109 | original = aw.run_mock_agent
|
| 110 | homes = []
|
| 111 | def observing_agent(*args, **kwargs):
|
| 112 | home = Path(kwargs["env"]["HOME"])
|
| 113 | homes.append(home)
|
| 114 | (home / "synthetic-provider-secret").write_text("private runtime state")
|
| 115 | return original(*args, **kwargs)
|
| 116 | with measurement_lock("attribution-regression"), mock.patch.object(aw, 'run_mock_agent', side_effect=observing_agent):
|
| 117 | row = aw.run_one(self.agent, self.subject, self.workflow, fixture, 0,
|
| 118 | trial / 'work', trial / 'artifacts', 30, 'agent-default', 'minimal', 'full-docs',
|
| 119 | 'test', '2026-09-18T00:00:00Z', {}, {})
|
| 120 | self.assertTrue(row['success'], row.get('failure_reason'))
|
| 121 | with mock.patch.object(agent_campaign, 'file_hash', wraps=agent_campaign.file_hash) as hashes:
|
| 122 | observed = agent_campaign.artifact_hashes(trial)
|
| 123 | self.assertTrue(observed)
|
| 124 | self.assertFalse(any('synthetic-provider-secret' in name for name in observed))
|
| 125 | self.assertFalse(any(Path(call.args[0]).is_relative_to(homes[0]) for call in hashes.call_args_list))
|
| 126 |
|
| 127 | def test_agent_home_content_is_outside_artifact_hash_tree(self):
|
| 128 | trial = self.root/'trial'
|
| 129 | artifacts = trial/'artifacts'
|
| 130 | artifacts.mkdir(parents=True)
|
| 131 | env, _ = aw.trial_agent_env(artifacts)
|
| 132 | home = Path(env['HOME'])
|
| 133 | self.addCleanup(shutil.rmtree, home, True)
|
| 134 | (home/'provider-secret-filename').write_text('synthetic-private-data')
|
| 135 | (artifacts/'prompt.txt').write_text('task')
|
| 136 | self.assertFalse(home.is_relative_to(artifacts))
|
| 137 | with mock.patch.object(agent_campaign, 'file_hash', wraps=agent_campaign.file_hash) as hashes:
|
| 138 | observed = agent_campaign.artifact_hashes(trial)
|
| 139 | self.assertEqual(set(observed), {'artifacts/prompt.txt'})
|
| 140 | self.assertFalse(any(Path(call.args[0]).is_relative_to(home) for call in hashes.call_args_list))
|