Viewing files at commit c10cefe7abe5Commit detailsView latest

oakmark/oak-engine/src/main.rs 541 lines · 18.8 KB · rust
1
//! oak-engine — the protocol-transport push engine for the oakmark throughput
2
//! benchmark.
3
//!
4
//! One process is spawned per synthetic agent by the Go harness and driven
5
//! serially over line-delimited JSON on stdin/stdout. Each command produces
6
//! exactly one JSON response line (flushed immediately). All diagnostics go to
7
//! stderr so stdout stays a clean JSON stream.
8
//!
9
//! It reuses `oak_core` to build blobs / trees / commits so every content
10
//! address is correct by construction (the server re-verifies them), and
11
//! hand-writes the HTTP push/pull against `oak_core::protocol` DTOs. It never
12
//! shells out to the `oak` CLI.
13
14
use std::io::{BufRead, Write};
15
16
use base64::engine::general_purpose::STANDARD as B64;
17
use base64::Engine as _;
18
use oak_core::protocol::{
19
    self, BlobCheckRequest, BlobCheckResponse, BlobData, BranchPushData, CommitData, FileChangeData,
20
    PullResponse, PushRequest, PushResponse,
21
};
22
use oak_core::{
23
    build_tree, Blob, ChangeType, Commit, FileChange, FileMode, Hash, ManifestEntry,
24
};
25
use serde_json::{json, Value};
26
27
/// Per-agent engine state, filled in by `init` and mutated as pushes land.
28
struct Engine {
29
    http: reqwest::blocking::Client,
30
    remote: String, // no trailing slash
31
    owner: String,
32
    name: String,
33
    token: String, // empty => anonymous (no auth header)
34
    author: String,
35
    branch: String,
36
37
    /// The branch's last-known remote head (None until the branch has a
38
    /// commit on the server). Used for the CAS `expected_*` fields.
39
    branch_head: Option<String>,
40
    /// Parent commit hash for the next commit we build. Seeded from the branch
41
    /// head if the branch exists, else from `main`'s head (the fork point), and
42
    /// advanced to each accepted commit so the branch is a proper chain.
43
    parent_next: Option<String>,
44
    /// Full path -> blob-hash snapshot of the current branch head, so we can
45
    /// emit accurate Added/Modified/Deleted file changes on the next push.
46
    prev_files: std::collections::HashMap<String, String>,
47
}
48
49
impl Engine {
50
    fn auth<'a>(&self, rb: reqwest::blocking::RequestBuilder) -> reqwest::blocking::RequestBuilder {
51
        if self.token.is_empty() {
52
            rb
53
        } else {
54
            rb.header("authorization", format!("Bearer {}", self.token))
55
        }
56
    }
57
58
    /// `GET /pull?branch_name=<b>&force=true` and return the parsed response.
59
    fn pull(&self, branch_name: &str) -> Result<PullResponse, String> {
60
        let url = format!(
61
            "{}/api/{}/{}/pull?branch_name={}&force=true",
62
            self.remote, self.owner, self.name, branch_name
63
        );
64
        let resp = self
65
            .auth(self.http.get(&url))
66
            .send()
67
            .map_err(|e| format!("pull transport error: {e}"))?;
68
        let status = resp.status();
69
        if !status.is_success() {
70
            return Err(format!("pull {branch_name} returned HTTP {}", status.as_u16()));
71
        }
72
        resp.json::<PullResponse>()
73
            .map_err(|e| format!("pull decode error: {e}"))
74
    }
75
}
76
77
/// One decoded file from a push command.
78
struct InFile {
79
    path: String,
80
    content: Vec<u8>,
81
}
82
83
fn main() {
84
    let stdin = std::io::stdin();
85
    let mut out = std::io::stdout();
86
    let mut engine: Option<Engine> = None;
87
88
    for line in stdin.lock().lines() {
89
        let line = match line {
90
            Ok(l) => l,
91
            Err(e) => {
92
                eprintln!("oak-engine: stdin read error: {e}");
93
                break;
94
            }
95
        };
96
        let trimmed = line.trim();
97
        if trimmed.is_empty() {
98
            continue;
99
        }
100
101
        let cmd: Value = match serde_json::from_str(trimmed) {
102
            Ok(v) => v,
103
            Err(e) => {
104
                write_line(&mut out, &json!({"ok": false, "err": format!("bad json: {e}")}));
105
                continue;
106
            }
107
        };
108
109
        let op = cmd.get("op").and_then(Value::as_str).unwrap_or("");
110
        match op {
111
            "init" => {
112
                let resp = do_init(&cmd);
113
                match resp {
114
                    Ok(e) => {
115
                        engine = Some(e);
116
                        write_line(&mut out, &json!({"ok": true}));
117
                    }
118
                    Err(err) => write_line(&mut out, &json!({"ok": false, "err": err})),
119
                }
120
            }
121
            "push" => {
122
                let resp = match engine.as_mut() {
123
                    Some(e) => do_push(e, &cmd),
124
                    None => json!({"ok": false, "cas": false, "new_head": Value::Null, "err": "push before init"}),
125
                };
126
                write_line(&mut out, &resp);
127
            }
128
            "clone" => {
129
                let resp = match engine.as_ref() {
130
                    Some(e) => do_clone(e),
131
                    None => json!({"ok": false, "clone_ms": 0.0, "err": "clone before init"}),
132
                };
133
                write_line(&mut out, &resp);
134
            }
135
            "quit" => {
136
                std::process::exit(0);
137
            }
138
            other => {
139
                write_line(
140
                    &mut out,
141
                    &json!({"ok": false, "err": format!("unknown op: {other}")}),
142
                );
143
            }
144
        }
145
    }
146
}
147
148
fn write_line(out: &mut std::io::Stdout, v: &Value) {
149
    // One compact JSON object per line, flushed so the harness sees it
150
    // immediately (it drives us request/response, serially).
151
    let _ = writeln!(out, "{v}");
152
    let _ = out.flush();
153
}
154
155
fn do_init(cmd: &Value) -> Result<Engine, String> {
156
    let get = |k: &str| cmd.get(k).and_then(Value::as_str).unwrap_or("").to_string();
157
    let remote = get("remote");
158
    let remote = remote.trim_end_matches('/').to_string();
159
    let owner = get("owner");
160
    let name = get("name");
161
    let token = get("token");
162
    let author = get("author");
163
    let branch = get("branch");
164
    if remote.is_empty() || owner.is_empty() || name.is_empty() || branch.is_empty() {
165
        return Err("init requires remote, owner, name, branch".to_string());
166
    }
167
168
    let http = reqwest::blocking::Client::builder()
169
        .build()
170
        .map_err(|e| format!("http client build: {e}"))?;
171
172
    let mut engine = Engine {
173
        http,
174
        remote,
175
        owner,
176
        name,
177
        token,
178
        author,
179
        branch,
180
        branch_head: None,
181
        parent_next: None,
182
        prev_files: std::collections::HashMap::new(),
183
    };
184
185
    // Does our branch already exist on the server (has a head)? If so we chain
186
    // onto it; otherwise we fork from main's head.
187
    let branch_pull = engine.pull(&engine.branch)?;
188
    if let Some(head) = branch_pull.head.clone() {
189
        engine.branch_head = Some(head.clone());
190
        engine.parent_next = Some(head);
191
        eprintln!("oak-engine[{}]: branch exists, head={:?}", engine.branch, engine.branch_head);
192
    } else {
193
        let main_pull = engine.pull("main")?;
194
        engine.branch_head = None;
195
        engine.parent_next = main_pull.head.clone();
196
        eprintln!(
197
            "oak-engine[{}]: new branch, forking from main head={:?}",
198
            engine.branch, engine.parent_next
199
        );
200
    }
201
202
    Ok(engine)
203
}
204
205
fn do_push(engine: &mut Engine, cmd: &Value) -> Value {
206
    match build_and_push(engine, cmd) {
207
        Ok(PushOutcome::Ok { new_head }) => {
208
            json!({"ok": true, "cas": false, "new_head": new_head, "err": Value::Null})
209
        }
210
        Ok(PushOutcome::Cas { new_head, message }) => {
211
            json!({"ok": false, "cas": true, "new_head": new_head, "err": message})
212
        }
213
        Err(err) => {
214
            json!({"ok": false, "cas": false, "new_head": Value::Null, "err": err})
215
        }
216
    }
217
}
218
219
enum PushOutcome {
220
    Ok { new_head: String },
221
    Cas { new_head: Value, message: String },
222
}
223
224
fn build_and_push(engine: &mut Engine, cmd: &Value) -> Result<PushOutcome, String> {
225
    let files = decode_files(cmd)?;
226
227
    // 1. Blobs + manifest entries. Blob::new computes the BLAKE3 content hash
228
    //    the server re-verifies.
229
    let mut entries: Vec<ManifestEntry> = Vec::with_capacity(files.len());
230
    // Unique blob hash -> content, for the push payload.
231
    let mut blob_by_hash: std::collections::HashMap<String, (Vec<u8>, u64)> =
232
        std::collections::HashMap::new();
233
    // The new full snapshot: path -> blob hash.
234
    let mut new_files: std::collections::HashMap<String, String> =
235
        std::collections::HashMap::new();
236
    for f in &files {
237
        let blob = Blob::new(f.content.clone());
238
        let hh = blob.hash.to_string();
239
        entries.push(ManifestEntry {
240
            path: f.path.clone(),
241
            blob_hash: blob.hash.clone(),
242
            mode: FileMode::Regular,
243
        });
244
        new_files.insert(f.path.clone(), hh.clone());
245
        blob_by_hash
246
            .entry(hh)
247
            .or_insert((blob.content, blob.size));
248
    }
249
250
    // 2. Tree objects (root hash == the commit's manifest_hash).
251
    let built = build_tree(&entries).map_err(|e| format!("build_tree: {e}"))?;
252
    let manifest_hash = built.root_hash.clone();
253
254
    // 3. File changes vs the previous snapshot (Added/Modified/Deleted). The v1
255
    //    commit hash excludes this list, but the server stores and validates the
256
    //    individual rows, so we send accurate ones.
257
    let file_changes = diff_snapshots(&engine.prev_files, &new_files);
258
259
    // 4. Commit (computes its own verified hash from the truncated-micros now()).
260
    let parent_hash = engine
261
        .parent_next
262
        .as_ref()
263
        .map(|s| Hash::from_hex(s))
264
        .transpose()
265
        .map_err(|e| format!("bad parent hash: {e}"))?;
266
    let commit = Commit::new(
267
        engine.branch.clone(),
268
        parent_hash,
269
        None,
270
        manifest_hash,
271
        engine.author.clone(),
272
        None,
273
        file_changes,
274
    )
275
    .map_err(|e| format!("Commit::new: {e}"))?;
276
277
    let commit_data = commit_to_wire(&commit);
278
279
    // 5. Blob payload — optionally drop blobs the server already has.
280
    let mut blobs: Vec<BlobData> = blob_by_hash
281
        .into_iter()
282
        .map(|(hash, (content, size))| BlobData {
283
            hash,
284
            content,
285
            size,
286
            chunks: Vec::new(),
287
        })
288
        .collect();
289
    maybe_drop_present_blobs(engine, &mut blobs);
290
291
    // 6. Trees on the wire.
292
    let trees = built.trees.iter().map(protocol::tree_to_wire).collect();
293
294
    // 7. Branch metadata (flat model: parent=main, open).
295
    let branch_data = BranchPushData {
296
        name: engine.branch.clone(),
297
        description: None,
298
        parent_branch: Some("main".to_string()),
299
        status: "open".to_string(),
300
        created_at: chrono::Utc::now().to_rfc3339(),
301
        close_reason: None,
302
    };
303
304
    // 8. The push request. force=true (each agent owns its branch), but we still
305
    //    send the tracked heads so the chain is explicit.
306
    let push_req = PushRequest {
307
        expected_head: engine.branch_head.clone(),
308
        expected_branch_head: engine.branch_head.clone(),
309
        force: true,
310
        branch: Some(branch_data),
311
        commits: vec![commit_data],
312
        blobs,
313
        trees,
314
    };
315
316
    let url = format!("{}/api/{}/{}/push", engine.remote, engine.owner, engine.name);
317
    let resp = engine
318
        .auth(
319
            engine
320
                .http
321
                .post(&url)
322
                .header("x-oak-user", engine.author.clone())
323
                .json(&push_req),
324
        )
325
        .send()
326
        .map_err(|e| format!("push transport error: {e}"))?;
327
328
    let status = resp.status();
329
    if !status.is_success() {
330
        let body = resp.text().unwrap_or_default();
331
        return Err(format!("push HTTP {}: {}", status.as_u16(), truncate(&body, 300)));
332
    }
333
334
    let pr: PushResponse = resp
335
        .json()
336
        .map_err(|e| format!("push decode error: {e}"))?;
337
338
    if pr.success {
339
        let new_head = pr
340
            .new_head
341
            .clone()
342
            .unwrap_or_else(|| commit.hash.to_string());
343
        // Advance tracked state only on an accepted push.
344
        engine.branch_head = Some(new_head.clone());
345
        engine.parent_next = Some(new_head.clone());
346
        engine.prev_files = new_files;
347
        Ok(PushOutcome::Ok { new_head })
348
    } else {
349
        // HTTP 200 + success:false is a CAS/conflict, not an error. Leave state
350
        // untouched so the caller can retry against the real head.
351
        Ok(PushOutcome::Cas {
352
            new_head: pr
353
                .new_head
354
                .map(Value::from)
355
                .unwrap_or(Value::Null),
356
            message: format!("success:false: {}", pr.message),
357
        })
358
    }
359
}
360
361
/// Best-effort `POST /blobs/check` to drop blobs the server already has. Skipped
362
/// when the inline payload is below one round trip's worth of bytes (64 KiB).
363
/// Any failure falls through to "send everything," which is always correct.
364
fn maybe_drop_present_blobs(engine: &Engine, blobs: &mut Vec<BlobData>) {
365
    if blobs.is_empty() {
366
        return;
367
    }
368
    const MIN_BYTES: u64 = 64 * 1024;
369
    let inline: u64 = blobs.iter().map(|b| b.content.len() as u64).sum();
370
    if inline < MIN_BYTES {
371
        eprintln!(
372
            "oak-engine[{}]: skipping blobs/check ({} blob(s), {inline} B < {MIN_BYTES})",
373
            engine.branch,
374
            blobs.len()
375
        );
376
        return;
377
    }
378
    let hashes: Vec<String> = blobs.iter().map(|b| b.hash.clone()).collect();
379
    let url = format!(
380
        "{}/api/{}/{}/blobs/check",
381
        engine.remote, engine.owner, engine.name
382
    );
383
    let req = BlobCheckRequest { hashes };
384
    match engine.auth(engine.http.post(&url).json(&req)).send() {
385
        Ok(resp) if resp.status().is_success() => match resp.json::<BlobCheckResponse>() {
386
            Ok(body) => {
387
                let missing: std::collections::HashSet<String> =
388
                    body.missing.into_iter().collect();
389
                let before = blobs.len();
390
                blobs.retain(|b| missing.contains(&b.hash));
391
                eprintln!(
392
                    "oak-engine[{}]: blobs/check kept {}/{}",
393
                    engine.branch,
394
                    blobs.len(),
395
                    before
396
                );
397
            }
398
            Err(e) => eprintln!("oak-engine: blobs/check decode failed ({e}); sending all"),
399
        },
400
        Ok(resp) => eprintln!(
401
            "oak-engine: blobs/check HTTP {}; sending all",
402
            resp.status().as_u16()
403
        ),
404
        Err(e) => eprintln!("oak-engine: blobs/check request failed ({e}); sending all"),
405
    }
406
}
407
408
fn do_clone(engine: &Engine) -> Value {
409
    let t0 = std::time::Instant::now();
410
    match engine.pull("main") {
411
        Ok(pr) => {
412
            let ms = t0.elapsed().as_secs_f64() * 1000.0;
413
            eprintln!(
414
                "oak-engine[{}]: clone (pull main) head={:?} commits={} blobs={} trees={} in {ms:.3}ms",
415
                engine.branch,
416
                pr.head,
417
                pr.commits.len(),
418
                pr.blobs.len(),
419
                pr.trees.len()
420
            );
421
            json!({"ok": true, "clone_ms": ms, "err": Value::Null})
422
        }
423
        Err(e) => {
424
            let ms = t0.elapsed().as_secs_f64() * 1000.0;
425
            json!({"ok": false, "clone_ms": ms, "err": e})
426
        }
427
    }
428
}
429
430
fn decode_files(cmd: &Value) -> Result<Vec<InFile>, String> {
431
    let arr = cmd
432
        .get("files")
433
        .and_then(Value::as_array)
434
        .ok_or_else(|| "push requires a files array".to_string())?;
435
    let mut out = Vec::with_capacity(arr.len());
436
    for f in arr {
437
        let path = f
438
            .get("path")
439
            .and_then(Value::as_str)
440
            .ok_or_else(|| "file missing path".to_string())?
441
            .to_string();
442
        let b64 = f
443
            .get("content_b64")
444
            .and_then(Value::as_str)
445
            .ok_or_else(|| "file missing content_b64".to_string())?;
446
        let content = B64
447
            .decode(b64)
448
            .map_err(|e| format!("bad base64 for {path}: {e}"))?;
449
        out.push(InFile { path, content });
450
    }
451
    Ok(out)
452
}
453
454
/// Diff two path->blob-hash snapshots into FileChanges. Unchanged files produce
455
/// no change (matching how `oak commit` records only real changes).
456
fn diff_snapshots(
457
    prev: &std::collections::HashMap<String, String>,
458
    next: &std::collections::HashMap<String, String>,
459
) -> Vec<FileChange> {
460
    let mut changes = Vec::new();
461
    for (path, new_hash) in next {
462
        match prev.get(path) {
463
            None => changes.push(FileChange {
464
                path: path.clone(),
465
                change_type: ChangeType::Added,
466
                old_blob_hash: None,
467
                new_blob_hash: Some(Hash(new_hash.clone())),
468
                old_path: None,
469
                old_mode: None,
470
                new_mode: None,
471
            }),
472
            Some(old_hash) if old_hash != new_hash => changes.push(FileChange {
473
                path: path.clone(),
474
                change_type: ChangeType::Modified,
475
                old_blob_hash: Some(Hash(old_hash.clone())),
476
                new_blob_hash: Some(Hash(new_hash.clone())),
477
                old_path: None,
478
                old_mode: None,
479
                new_mode: None,
480
            }),
481
            Some(_) => {} // unchanged
482
        }
483
    }
484
    for (path, old_hash) in prev {
485
        if !next.contains_key(path) {
486
            changes.push(FileChange {
487
                path: path.clone(),
488
                change_type: ChangeType::Deleted,
489
                old_blob_hash: Some(Hash(old_hash.clone())),
490
                new_blob_hash: None,
491
                old_path: None,
492
                old_mode: None,
493
                new_mode: None,
494
            });
495
        }
496
    }
497
    changes
498
}
499
500
/// Convert a core `Commit` to its wire `CommitData`. Mirrors the CLI's push
501
/// path exactly so the server reproduces the same commit hash.
502
fn commit_to_wire(c: &Commit) -> CommitData {
503
    CommitData {
504
        hash: c.hash.to_string(),
505
        branch_name: c.branch_name.clone(),
506
        parent_hash: c.parent_hash.as_ref().map(|h| h.to_string()),
507
        merge_parent_hash: c.merge_parent_hash.as_ref().map(|h| h.to_string()),
508
        manifest_hash: c.manifest_hash.to_string(),
509
        author: c.author.clone(),
510
        message: c.message.clone(),
511
        timestamp: c.timestamp.to_rfc3339(),
512
        files: c
513
            .files
514
            .iter()
515
            .map(|f| FileChangeData {
516
                path: f.path.clone(),
517
                change_type: match f.change_type {
518
                    ChangeType::Added => "added".to_string(),
519
                    ChangeType::Modified => "modified".to_string(),
520
                    ChangeType::Deleted => "deleted".to_string(),
521
                    ChangeType::Renamed => "renamed".to_string(),
522
                },
523
                old_blob_hash: f.old_blob_hash.as_ref().map(|h| h.to_string()),
524
                new_blob_hash: f.new_blob_hash.as_ref().map(|h| h.to_string()),
525
                old_path: f.old_path.clone(),
526
            })
527
            .collect(),
528
    }
529
}
530
531
fn truncate(s: &str, n: usize) -> String {
532
    if s.len() <= n {
533
        return s.to_string();
534
    }
535
    // Back off to the nearest char boundary so we never slice mid-codepoint.
536
    let mut end = n;
537
    while end > 0 && !s.is_char_boundary(end) {
538
        end -= 1;
539
    }
540
    format!("{}…", &s[..end])
541
}