Log in
benchmarksPublic

Viewing files at commit ef78687eef1aCommit detailsView latest

Browse files
tests/test_netshape_bench.py 561 lines · 22.6 KB · python
1
"""Lane CLI tests: on a host that cannot shape packets (this macOS dev host),
2
netshape_bench must emit honest skip rows for every requested run and exit 3
3
— never error. Also covers usage errors, the --help pressure quote, and the
4
oak served-repo seed/clone-guard helpers (unit-tested directly because the
5
Linux measurement path is unreachable here)."""
6
7
from __future__ import annotations
8
9
import json
10
import os
11
import shutil
12
import subprocess
13
import sys
14
import tempfile
15
import unittest
16
from pathlib import Path
17
18
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
19
20
import netshape_bench  # noqa: E402
21
from oakbench.rows import validate_rows  # noqa: E402
22
23
ROOT = Path(__file__).resolve().parents[1]
24
SCRIPT = ROOT / "scripts" / "netshape_bench.py"
25
GIT = shutil.which("git")
26
27
CAN_ACTUALLY_SHAPE = sys.platform == "linux" and hasattr(os, "geteuid") and os.geteuid() == 0
28
29
REQUIRED_ROW_KEYS = {
30
    "bench_id",
31
    "profile",
32
    "scenario",
33
    "operation",
34
    "run",
35
    "subject",
36
    "subject_kind",
37
    "elapsed_ms",
38
    "returncode",
39
    "command",
40
    "netshape_profile",
41
    "remote_transport",
42
    "remote_server",
43
}
44
45
46
def run_lane(args: list[str]) -> subprocess.CompletedProcess:
47
    return subprocess.run(
48
        [sys.executable, str(SCRIPT), *args],
49
        capture_output=True,
50
        text=True,
51
        timeout=120,
52
    )
53
54
55
class NetshapeBenchSkipPathTest(unittest.TestCase):
56
    @unittest.skipIf(CAN_ACTUALLY_SHAPE, "host has CAP_NET_ADMIN; skip path not reachable")
57
    def test_unavailable_host_emits_skip_rows_and_exits_3(self) -> None:
58
        with tempfile.TemporaryDirectory() as tmp:
59
            out = Path(tmp) / "rows.jsonl"
60
            proc = run_lane(
61
                [
62
                    "--profile", "wan-30ms",
63
                    "--scenario", "netshape_clone_cold",
64
                    "--subjects", "git,oak",
65
                    "--results", str(out),
66
                    "--runs", "2",
67
                ]
68
            )
69
            self.assertEqual(proc.returncode, 3, proc.stderr)
70
            rows = [json.loads(line) for line in out.read_text().splitlines()]
71
        # 2 subjects x 2 runs, every one an honest skip row.
72
        self.assertEqual(len(rows), 4)
73
        self.assertEqual({row["subject"] for row in rows}, {"git", "oak"})
74
        self.assertEqual({row["run"] for row in rows}, {0, 1})
75
        for row in rows:
76
            self.assertTrue(REQUIRED_ROW_KEYS.issubset(row.keys()), sorted(row.keys()))
77
            self.assertTrue(row["skipped"])
78
            self.assertEqual(row["returncode"], 77)
79
            self.assertTrue(row["skip_reason"].startswith("netshape_unavailable:"), row["skip_reason"])
80
            self.assertEqual(row["netshape_profile"], "wan-30ms")
81
            self.assertEqual(row["remote_transport"], "network_shaped")
82
            self.assertEqual(row["remote_server"], "netshape-local")
83
            self.assertEqual(row["scenario"], "netshape_clone_cold")
84
            self.assertEqual(row["elapsed_ms"], 0.0)
85
            self.assertEqual(row["command"], [])
86
87
    @unittest.skipIf(CAN_ACTUALLY_SHAPE, "host has CAP_NET_ADMIN; skip path not reachable")
88
    def test_results_directory_form_and_every_scenario_skips(self) -> None:
89
        with tempfile.TemporaryDirectory() as tmp:
90
            proc = run_lane(
91
                [
92
                    "--profile", "cellular-300ms-jitter",
93
                    "--scenario", "netshape_task_loop",
94
                    "--subjects", "oak",
95
                    "--results", tmp,
96
                    "--runs", "1",
97
                ]
98
            )
99
            self.assertEqual(proc.returncode, 3, proc.stderr)
100
            written = list(Path(tmp).glob("netshape-*.jsonl"))
101
            self.assertEqual(len(written), 1)
102
            rows = [json.loads(line) for line in written[0].read_text().splitlines()]
103
        self.assertEqual(len(rows), 1)
104
        self.assertEqual(rows[0]["subject"], "oak")
105
        self.assertEqual(rows[0]["netshape_profile"], "cellular-300ms-jitter")
106
        self.assertTrue(rows[0]["skip_reason"].startswith("netshape_unavailable:"))
107
108
    @unittest.skipIf(CAN_ACTUALLY_SHAPE, "host has CAP_NET_ADMIN; skip path not reachable")
109
    def test_log_follow_scenario_skips_honestly_when_netshape_unavailable(self) -> None:
110
        with tempfile.TemporaryDirectory() as tmp:
111
            out = Path(tmp) / "rows.jsonl"
112
            proc = run_lane(
113
                [
114
                    "--profile", "wan-30ms",
115
                    "--scenario", "netshape_log_follow_cold",
116
                    "--subjects", "git,oak",
117
                    "--results", str(out),
118
                    "--runs", "1",
119
                ]
120
            )
121
            self.assertEqual(proc.returncode, 3, proc.stderr)
122
            rows = [json.loads(line) for line in out.read_text().splitlines()]
123
        self.assertEqual({row["scenario"] for row in rows}, {"netshape_log_follow_cold"})
124
        self.assertEqual({row["subject"] for row in rows}, {"git", "oak"})
125
        for row in rows:
126
            self.assertTrue(row["skipped"])
127
            self.assertEqual(row["returncode"], 77)
128
            self.assertTrue(row["skip_reason"].startswith("netshape_unavailable:"))
129
130
131
class NetshapeBenchUsageTest(unittest.TestCase):
132
    def test_local_server_template_is_parsed_as_argv_not_shell(self) -> None:
133
        argv = netshape_bench.local_server_argv(
134
            "oak serve --host {host} --port {port} --root '{root}' ; touch /tmp/owned",
135
            host="127.0.0.1",
136
            port=4321,
137
            root=Path("/tmp/root with spaces"),
138
        )
139
140
        self.assertEqual(
141
            argv,
142
            [
143
                "oak",
144
                "serve",
145
                "--host",
146
                "127.0.0.1",
147
                "--port",
148
                "4321",
149
                "--root",
150
                "/tmp/root with spaces",
151
                ";",
152
                "touch",
153
                "/tmp/owned",
154
            ],
155
        )
156
157
    def test_log_follow_is_named_scenario_with_scenario_skip_contract(self) -> None:
158
        self.assertIn("netshape_log_follow_cold", netshape_bench.SCENARIOS)
159
160
        meta = {
161
            "bench_id": "test",
162
            "profile": "netshape",
163
            "netshape_profile": "wan-30ms",
164
            "remote_transport": "network_shaped",
165
            "remote_server": "netshape-local",
166
        }
167
        row = netshape_bench.skip_row(
168
            meta,
169
            "netshape_log_follow_cold",
170
            "oak",
171
            0,
172
            "oak_local_server_missing",
173
        )
174
        self.assertEqual(row["scenario"], "netshape_log_follow_cold")
175
        self.assertEqual(row["operation"], "scenario.skip")
176
        self.assertEqual(row["returncode"], 77)
177
        self.assertEqual(validate_rows([row], "core"), [])
178
179
    def test_unknown_profile_exits_2(self) -> None:
180
        with tempfile.TemporaryDirectory() as tmp:
181
            proc = run_lane(["--profile", "warp-speed", "--results", tmp])
182
        self.assertEqual(proc.returncode, 2)
183
        self.assertIn("warp-speed", proc.stderr)
184
        self.assertIn("wan-30ms", proc.stderr)  # names the known profiles
185
186
    def test_unknown_subject_exits_2(self) -> None:
187
        with tempfile.TemporaryDirectory() as tmp:
188
            proc = run_lane(["--subjects", "git,hg", "--results", tmp])
189
        self.assertEqual(proc.returncode, 2)
190
191
    def test_unknown_scenario_exits_2(self) -> None:
192
        with tempfile.TemporaryDirectory() as tmp:
193
            proc = run_lane(["--scenario", "netshape_warp", "--results", tmp])
194
        self.assertEqual(proc.returncode, 2)
195
196
    def test_help_quotes_the_pressure(self) -> None:
197
        proc = run_lane(["--help"])
198
        self.assertEqual(proc.returncode, 0)
199
        self.assertIn("the skip is the pressure", proc.stdout)
200
        self.assertIn("OAK_BENCH_LOCAL_SERVER_CMD", proc.stdout)
201
        self.assertIn("oak_local_server_missing", proc.stdout)
202
203
204
class FakeRunner:
205
    """Injectable runner for seed_oak_served_repo: records calls, fails one stage."""
206
207
    def __init__(self, fail_stage: str | None = None) -> None:
208
        self.fail_stage = fail_stage
209
        self.calls: list[tuple[list[str], Path | None]] = []
210
211
    def __call__(self, command, cwd=None, timeout=600.0):
212
        self.calls.append((list(command), cwd))
213
        failed = self.fail_stage is not None and command[1] == self.fail_stage
214
        return subprocess.CompletedProcess(
215
            command, 1 if failed else 0, stdout="", stderr="boom" if failed else ""
216
        )
217
218
219
class SeedOakServedRepoTest(unittest.TestCase):
220
    """The local oak server starts over an EMPTY served_root; the seed helper
221
    must create the served repo (init -> commit -> push to the server URL)
222
    before any measured run, and name the failed stage when it cannot."""
223
224
    URL = "http://127.0.0.1:9999/bench"
225
226
    def test_successful_seed_runs_init_commit_push_in_seed_dir(self) -> None:
227
        runner = FakeRunner()
228
        with tempfile.TemporaryDirectory() as tmp:
229
            workdir = Path(tmp)
230
            ok, reason = netshape_bench.seed_oak_served_repo("oak", self.URL, workdir, runner=runner)
231
            seed = workdir / "oak-seed-src"
232
            self.assertTrue((seed / "README.md").exists())
233
        self.assertTrue(ok)
234
        self.assertEqual(reason, "")
235
        self.assertEqual(
236
            [command for command, _ in runner.calls],
237
            [
238
                ["oak", "init", "."],
239
                ["oak", "commit", "--no-verify"],
240
                ["oak", "push", self.URL],
241
            ],
242
        )
243
        # Every stage runs inside the seed workdir, mirroring the task_loop
244
        # git-seed idiom (init a scratch repo, commit, push to the URL).
245
        self.assertEqual({cwd for _, cwd in runner.calls}, {seed})
246
247
    def test_failed_stage_names_the_stage_and_stops(self) -> None:
248
        for stage, expected_calls in (("init", 1), ("commit", 2), ("push", 3)):
249
            runner = FakeRunner(fail_stage=stage)
250
            with tempfile.TemporaryDirectory() as tmp:
251
                ok, reason = netshape_bench.seed_oak_served_repo(
252
                    "oak", self.URL, Path(tmp), runner=runner
253
                )
254
            self.assertFalse(ok, stage)
255
            self.assertEqual(reason, f"oak_server_seed_failed:{stage}")
256
            self.assertEqual(len(runner.calls), expected_calls, stage)
257
258
259
class PreconditionCloneGuardTest(unittest.TestCase):
260
    """A failed precondition clone must become one recorded failure row plus
261
    a prior_step_failed:clone skip row per dependent step — never a crash
262
    into a missing dest."""
263
264
    META = {
265
        "bench_id": "20260612T000000Z",
266
        "profile": "netshape",
267
        "netshape_profile": "wan-30ms",
268
        "remote_transport": "network_shaped",
269
        "remote_server": "netshape-local",
270
    }
271
272
    def test_successful_clone_returns_none(self) -> None:
273
        with tempfile.TemporaryDirectory() as tmp:
274
            dest = Path(tmp)  # exists
275
            proc = subprocess.CompletedProcess(["oak", "clone"], 0, stdout="", stderr="")
276
            self.assertIsNone(
277
                netshape_bench.precondition_clone_failed_rows(
278
                    self.META, "netshape_pull_uptodate", "oak", 0,
279
                    ["oak", "clone", "url", str(dest)], proc, dest, ["oak.pull"],
280
                )
281
            )
282
283
    def test_failed_clone_cascades_into_skip_rows(self) -> None:
284
        with tempfile.TemporaryDirectory() as tmp:
285
            dest = Path(tmp) / "missing-dest"
286
            command = ["oak", "clone", "url", str(dest)]
287
            proc = subprocess.CompletedProcess(command, 128, stdout="", stderr="fatal: repo not found")
288
            rows = netshape_bench.precondition_clone_failed_rows(
289
                self.META, "netshape_task_loop", "oak", 1, command, proc, dest,
290
                ["loop.push"] * netshape_bench.TASK_LOOP_ITERATIONS,
291
            )
292
        self.assertEqual(len(rows), 1 + netshape_bench.TASK_LOOP_ITERATIONS)
293
        failure = rows[0]
294
        self.assertEqual(failure["operation"], "oak.clone")
295
        self.assertEqual(failure["returncode"], 128)
296
        self.assertEqual(failure["command"], command)
297
        self.assertIn("repo not found", failure["stderr"])
298
        self.assertNotIn("skipped", failure)
299
        for row in rows[1:]:
300
            self.assertEqual(row["operation"], "loop.push")
301
            self.assertTrue(row["skipped"])
302
            self.assertEqual(row["returncode"], 77)
303
            self.assertEqual(row["skip_reason"], "prior_step_failed:clone")
304
            self.assertEqual(row["run"], 1)
305
            self.assertEqual(row["subject"], "oak")
306
        # The cascade satisfies the row contract netshape opts into.
307
        self.assertEqual(validate_rows(rows, "core"), [])
308
309
    def test_clone_exit_zero_without_dest_is_still_a_failure(self) -> None:
310
        with tempfile.TemporaryDirectory() as tmp:
311
            dest = Path(tmp) / "missing-dest"
312
            proc = subprocess.CompletedProcess(["git", "clone"], 0, stdout="", stderr="")
313
            rows = netshape_bench.precondition_clone_failed_rows(
314
                self.META, "netshape_push_small_delta", "git", 0,
315
                ["git", "clone", "url", str(dest)], proc, dest, ["git.push"],
316
            )
317
        self.assertEqual(len(rows), 2)
318
        self.assertEqual(rows[0]["returncode"], 1)
319
        self.assertIn("dest missing", rows[0]["stderr"])
320
        self.assertEqual(rows[1]["skip_reason"], "prior_step_failed:clone")
321
        self.assertEqual(validate_rows(rows, "core"), [])
322
323
324
@unittest.skipIf(GIT is None, "git binary not installed")
325
class NetshapeLogFollowHelpersTest(unittest.TestCase):
326
    def test_make_served_git_repo_can_seed_history_for_log_follow(self) -> None:
327
        with tempfile.TemporaryDirectory() as tmp:
328
            root = Path(tmp)
329
            bare = netshape_bench.make_served_git_repo(
330
                GIT,
331
                root / "work",
332
                root / "served",
333
                refs_count=0,
334
                history_commits=4,
335
            )
336
            count = subprocess.check_output(
337
                [GIT, "-C", str(bare), "rev-list", "--count", "HEAD"],
338
                text=True,
339
            ).strip()
340
        self.assertEqual(count, "4")
341
342
    def test_timed_sequence_row_records_combined_command_and_ttfd_fields(self) -> None:
343
        class FakeStats:
344
            client_to_upstream_bytes = 11
345
            upstream_to_client_bytes = 22
346
            total_bytes = 33
347
            connections = 1
348
            first_upstream_to_client_ms = 3.5
349
            first_payload_bytes_server_to_client = 7
350
351
        class FakeProxy:
352
            def __init__(self) -> None:
353
                self.reset = False
354
355
            def reset_counts(self) -> None:
356
                self.reset = True
357
358
            def snapshot(self) -> FakeStats:
359
                return FakeStats()
360
361
        with tempfile.TemporaryDirectory() as tmp:
362
            proxy = FakeProxy()
363
            args = type("Args", (), {"scenario": "netshape_log_follow_cold"})()
364
            row = netshape_bench.timed_sequence_row(
365
                {
366
                    "bench_id": "test",
367
                    "profile": "netshape",
368
                    "netshape_profile": "lan-1ms",
369
                    "remote_transport": "network_shaped",
370
                    "remote_server": "netshape-local",
371
                },
372
                args,
373
                "git",
374
                0,
375
                "git.log.follow",
376
                [["true"], ["true"]],
377
                Path(tmp),
378
                proxy,
379
                extra={"history_path": "src/main.py"},
380
            )
381
382
        self.assertTrue(proxy.reset)
383
        self.assertEqual(row["returncode"], 0)
384
        self.assertEqual(row["command"], [["true"], ["true"]])
385
        self.assertEqual(row["wire_bytes_total"], 33)
386
        self.assertEqual(row["ttfd_ms"], 3.5)
387
        self.assertEqual(row["ttfd_source"], "byteproxy_first_upstream_to_client_byte")
388
        self.assertEqual(row["first_payload_bytes_server_to_client"], 7)
389
        self.assertEqual(row["history_path"], "src/main.py")
390
        self.assertEqual(validate_rows([row], "core"), [])
391
392
    def test_log_follow_sequence_row_carries_semantics_and_ttfd_metadata(self) -> None:
393
        class FakeStats:
394
            client_to_upstream_bytes = 17
395
            upstream_to_client_bytes = 41
396
            total_bytes = 58
397
            connections = 1
398
            first_upstream_to_client_ms = 9.25
399
            first_payload_bytes_server_to_client = 13
400
401
        class FakeProxy:
402
            def reset_counts(self) -> None:
403
                pass
404
405
            def snapshot(self) -> FakeStats:
406
                return FakeStats()
407
408
        meta = {
409
            "bench_id": "test",
410
            "profile": "netshape",
411
            "netshape_profile": "wan-30ms",
412
            "remote_transport": "network_shaped",
413
            "remote_server": "netshape-local",
414
        }
415
        args = type("Args", (), {"scenario": "netshape_log_follow_cold"})()
416
        with tempfile.TemporaryDirectory() as tmp:
417
            row = netshape_bench.timed_sequence_row(
418
                meta,
419
                args,
420
                "git",
421
                0,
422
                "git.log.follow",
423
                [["true"], ["true"]],
424
                Path(tmp),
425
                FakeProxy(),
426
                extra={
427
                    "history_path": "src/main.py",
428
                    "history_commits_seeded": 8,
429
                    "ttfd_semantics": "cold_network_acquire_then_file_history_query",
430
                },
431
            )
432
433
        self.assertEqual(row["scenario"], "netshape_log_follow_cold")
434
        self.assertEqual(row["operation"], "git.log.follow")
435
        self.assertEqual(row["ttfd_ms"], 9.25)
436
        self.assertEqual(row["ttfd_source"], "byteproxy_first_upstream_to_client_byte")
437
        self.assertEqual(row["first_payload_bytes_server_to_client"], 13)
438
        self.assertEqual(row["ttfd_semantics"], "cold_network_acquire_then_file_history_query")
439
        self.assertEqual(validate_rows([row], "core"), [])
440
441
    def test_timed_sequence_row_carries_explicit_null_ttfd_keys_when_unobserved(self) -> None:
442
        """ADR-0002: a proxy that never saw an upstream-to-client payload byte
443
        (or no proxy at all) must yield explicit nulls, never absent keys."""
444
445
        class NoPayloadStats:
446
            client_to_upstream_bytes = 5
447
            upstream_to_client_bytes = 0
448
            total_bytes = 5
449
            connections = 1
450
            first_upstream_to_client_ms = None
451
            first_payload_bytes_server_to_client = None
452
453
        class NoPayloadProxy:
454
            def reset_counts(self) -> None:
455
                pass
456
457
            def snapshot(self) -> NoPayloadStats:
458
                return NoPayloadStats()
459
460
        meta = {
461
            "bench_id": "test",
462
            "profile": "netshape",
463
            "netshape_profile": "lan-1ms",
464
            "remote_transport": "network_shaped",
465
            "remote_server": "netshape-local",
466
        }
467
        args = type("Args", (), {"scenario": "netshape_log_follow_cold"})()
468
        with tempfile.TemporaryDirectory() as tmp:
469
            row = netshape_bench.timed_sequence_row(
470
                meta, args, "git", 0, "git.log.follow", [["true"]], Path(tmp), NoPayloadProxy(),
471
            )
472
            no_proxy_row = netshape_bench.timed_sequence_row(
473
                meta, args, "git", 0, "git.log.follow", [["true"]], Path(tmp), None,
474
            )
475
        for candidate in (row, no_proxy_row):
476
            for key in ("ttfd_ms", "ttfd_source", "first_payload_bytes_server_to_client"):
477
                self.assertIn(key, candidate)
478
                self.assertIsNone(candidate[key])
479
480
    def test_timed_row_carries_explicit_null_ttfd_keys_without_proxy(self) -> None:
481
        meta = {
482
            "bench_id": "test",
483
            "profile": "netshape",
484
            "netshape_profile": "lan-1ms",
485
            "remote_transport": "network_shaped",
486
            "remote_server": "netshape-local",
487
        }
488
        args = type(
489
            "Args", (), {"scenario": "netshape_log_follow_cold", "admitted_output_chars": 1000}
490
        )()
491
        with tempfile.TemporaryDirectory() as tmp:
492
            row = netshape_bench.timed_row(
493
                meta, args, "git", 0, "git.log.follow", ["true"], Path(tmp), None,
494
            )
495
        for key in ("ttfd_ms", "ttfd_source", "first_payload_bytes_server_to_client"):
496
            self.assertIn(key, row)
497
            self.assertIsNone(row[key])
498
        self.assertEqual(validate_rows([row], "core"), [])
499
500
501
class ExitCodeForRowsTest(unittest.TestCase):
502
    """The all-skipped verdict must look at SUBJECT MEASUREMENT rows only: a
503
    green netshape.selftest diagnostic row proves the pipe worked, not that
504
    anything was measured."""
505
506
    @staticmethod
507
    def selftest_row(returncode: int = 0) -> dict:
508
        return {
509
            "operation": "netshape.selftest",
510
            "subject": "harness",
511
            "subject_kind": "harness",
512
            "returncode": returncode,
513
        }
514
515
    @staticmethod
516
    def subject_skip(subject: str = "oak") -> dict:
517
        return {
518
            "operation": "scenario.skip",
519
            "subject": subject,
520
            "subject_kind": subject,
521
            "returncode": 77,
522
            "skipped": True,
523
            "skip_reason": "oak_local_server_missing",
524
        }
525
526
    @staticmethod
527
    def subject_measured(subject: str = "git", returncode: int = 0) -> dict:
528
        return {
529
            "operation": f"{subject}.clone",
530
            "subject": subject,
531
            "subject_kind": subject,
532
            "returncode": returncode,
533
        }
534
535
    def test_selftest_ok_plus_all_skipped_subjects_is_all_skipped(self) -> None:
536
        rows = [self.selftest_row(), self.subject_skip("git"), self.subject_skip("oak")]
537
        self.assertEqual(netshape_bench.exit_code_for_rows(rows), netshape_bench.EXIT_ALL_SKIPPED)
538
539
    def test_selftest_ok_plus_one_measured_subject_row_is_ok(self) -> None:
540
        rows = [self.selftest_row(), self.subject_measured("git"), self.subject_skip("oak")]
541
        self.assertEqual(netshape_bench.exit_code_for_rows(rows), netshape_bench.EXIT_OK)
542
543
    def test_selftest_only_is_all_skipped(self) -> None:
544
        rows = [self.selftest_row()]
545
        self.assertEqual(netshape_bench.exit_code_for_rows(rows), netshape_bench.EXIT_ALL_SKIPPED)
546
547
    def test_failures_check_unchanged(self) -> None:
548
        # A failing subject row dominates, even among skips.
549
        rows = [self.selftest_row(), self.subject_measured("git", returncode=1), self.subject_skip("oak")]
550
        self.assertEqual(netshape_bench.exit_code_for_rows(rows), netshape_bench.EXIT_FAILURES)
551
        # An out-of-tolerance self-test (returncode 1) is also a failure.
552
        rows = [self.selftest_row(returncode=1), self.subject_skip("git")]
553
        self.assertEqual(netshape_bench.exit_code_for_rows(rows), netshape_bench.EXIT_FAILURES)
554
555
    def test_all_skipped_without_selftest_still_exits_3(self) -> None:
556
        rows = [self.subject_skip("git"), self.subject_skip("oak")]
557
        self.assertEqual(netshape_bench.exit_code_for_rows(rows), netshape_bench.EXIT_ALL_SKIPPED)
558
559
560
if __name__ == "__main__":
561
    unittest.main()