use crate::{ CreateJavascriptContextRequest, JavascriptExecutionEngine, JavascriptExecutionError, StartJavascriptExecutionRequest,};use agentos_runtime::RuntimeContext;use serde::{Deserialize, Serialize};use std::collections::BTreeMap;use std::env;use std::fmt;use std::fmt::Write as _;use std::fs;use std::path::{Path, PathBuf};use std::process::Command;use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const BENCHMARK_MARKER_PREFIX: &str = "__AGENTOS_BENCH__:";const LOCAL_GRAPH_MODULE_COUNT: usize = 24;const BENCHMARK_ARTIFACT_VERSION: u32 = 5;const BENCHMARK_ARTIFACT_DIR: &str = "target/benchmark-reports/node-import-bench";const BENCHMARK_RUN_STATE_FILE: &str = "run-state.json";const TRANSPORT_RTT_CHANNEL: &str = "execution-stdio-echo";const TRANSPORT_RTT_PAYLOAD_BYTES: [usize; 3] = [32, 4 * 1024, 64 * 1024];const TRANSPORT_POLL_TIMEOUT: Duration = Duration::from_secs(5);const MAX_BENCHMARK_ITERATIONS: usize = 1_000;const MAX_BENCHMARK_WARMUP_ITERATIONS: usize = 1_000; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct JavascriptBenchmarkConfig { pub iterations: usize, pub warmup_iterations: usize,} impl Default for JavascriptBenchmarkConfig { fn default() -> Self { Self { iterations: 5, warmup_iterations: 1, } }} #[derive(Debug, Clone, PartialEq, Eq, Serialize)]pub struct BenchmarkHost { pub node_binary: String, pub node_version: String, pub os: &'static str, pub arch: &'static str, pub logical_cpus: usize,} #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]pub struct BenchmarkScenarioPhases<T> { pub context_setup_ms: T, pub startup_ms: T, #[serde(skip_serializing_if = "Option::is_none", default)] pub guest_execution_ms: Option<T>, pub completion_ms: T,} #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]pub struct BenchmarkStats { pub mean_ms: f64, pub p50_ms: f64, pub p95_ms: f64, pub min_ms: f64, pub max_ms: f64, pub stddev_ms: f64,} #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]pub struct BenchmarkDistributionStats { pub mean: f64, pub p50: f64, pub p95: f64, pub min: f64, pub max: f64, pub stddev: f64,} #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]pub struct BenchmarkResourceUsage<T> { #[serde(skip_serializing_if = "Option::is_none", default)] pub rss_bytes: Option<T>, #[serde(skip_serializing_if = "Option::is_none", default)] pub heap_used_bytes: Option<T>, #[serde(skip_serializing_if = "Option::is_none", default)] pub cpu_user_us: Option<T>, #[serde(skip_serializing_if = "Option::is_none", default)] pub cpu_system_us: Option<T>, #[serde(skip_serializing_if = "Option::is_none", default)] pub cpu_total_us: Option<T>,} #[derive(Debug, Clone, PartialEq, Serialize)]pub struct BenchmarkTransportRttReport { pub channel: &'static str, pub payload_bytes: usize, pub samples_ms: Vec<f64>, pub stats: BenchmarkStats,} #[derive(Debug, Clone, PartialEq, Serialize)]pub struct BenchmarkScenarioReport { pub id: &'static str, pub workload: &'static str, pub runtime: &'static str, pub mode: &'static str, pub description: &'static str, pub fixture: &'static str, pub compile_cache: &'static str, pub wall_samples_ms: Vec<f64>, pub wall_stats: BenchmarkStats, pub guest_import_samples_ms: Option<Vec<f64>>, pub guest_import_stats: Option<BenchmarkStats>, pub startup_overhead_samples_ms: Option<Vec<f64>>, pub startup_overhead_stats: Option<BenchmarkStats>, pub phase_samples_ms: BenchmarkScenarioPhases<Vec<f64>>, pub phase_stats: BenchmarkScenarioPhases<BenchmarkStats>, #[serde(skip_serializing_if = "Option::is_none", default)] pub resource_usage_samples: Option<BenchmarkResourceUsage<Vec<f64>>>, #[serde(skip_serializing_if = "Option::is_none", default)] pub resource_usage_stats: Option<BenchmarkResourceUsage<BenchmarkDistributionStats>>,} #[derive(Debug, Clone, PartialEq, Serialize)]pub struct JavascriptBenchmarkReport { pub generated_at_unix_ms: u128, pub config: JavascriptBenchmarkConfig, pub host: BenchmarkHost, pub repo_root: PathBuf, pub transport_rtt: Vec<BenchmarkTransportRttReport>, pub scenarios: Vec<BenchmarkScenarioReport>,} #[derive(Debug, Clone, PartialEq, Serialize)]pub struct BenchmarkComparison { pub baseline: BenchmarkComparisonBaseline, pub summary: BenchmarkComparisonSummary, pub scenario_deltas: Vec<BenchmarkScenarioDelta>, pub scenarios_missing_from_baseline: Vec<String>, pub baseline_only_scenarios: Vec<String>,} #[derive(Debug, Clone, PartialEq, Serialize)]pub struct BenchmarkComparisonBaseline { pub artifact_version: u32, pub generated_at_unix_ms: u128, pub path: PathBuf,} #[derive(Debug, Clone, PartialEq, Serialize)]pub struct BenchmarkComparisonSummary { pub compared_scenario_count: usize, #[serde(skip_serializing_if = "Option::is_none")] pub largest_wall_improvement: Option<BenchmarkDeltaHighlight>, #[serde(skip_serializing_if = "Option::is_none")] pub largest_wall_regression: Option<BenchmarkDeltaHighlight>,} #[derive(Debug, Clone, PartialEq, Serialize)]pub struct BenchmarkDeltaHighlight { pub id: String, pub delta_ms: f64, pub delta_pct: f64,} #[derive(Debug, Clone, PartialEq, Serialize)]pub struct BenchmarkScenarioDelta { pub id: String, pub description: String, pub wall_mean_ms: BenchmarkMetricDelta, #[serde(skip_serializing_if = "Option::is_none")] pub guest_import_mean_ms: Option<BenchmarkMetricDelta>, #[serde(skip_serializing_if = "Option::is_none")] pub startup_overhead_mean_ms: Option<BenchmarkMetricDelta>, #[serde(skip_serializing_if = "Option::is_none")] pub phase_mean_ms: Option<BenchmarkScenarioPhases<BenchmarkMetricDelta>>,} #[derive(Debug, Clone, PartialEq, Serialize)]pub struct BenchmarkMetricDelta { pub baseline_ms: f64, pub current_ms: f64, pub delta_ms: f64, pub delta_pct: f64,} impl JavascriptBenchmarkReport { pub fn render_markdown(&self) -> String { self.render_markdown_with_comparison(None) } pub fn render_markdown_with_comparison( &self, comparison: Option<&BenchmarkComparison>, ) -> String { let mut markdown = String::new(); let _ = writeln!(&mut markdown, "# secure-exec Node Import Benchmark"); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "- Generated at unix ms: `{}`", self.generated_at_unix_ms ); let _ = writeln!(&mut markdown, "- Node binary: `{}`", self.host.node_binary); let _ = writeln!( &mut markdown, "- Node version: `{}`", self.host.node_version.trim() ); let _ = writeln!( &mut markdown, "- Host: `{}` / `{}` / `{}` logical CPUs", self.host.os, self.host.arch, self.host.logical_cpus ); let _ = writeln!(&mut markdown, "- Repo root: `{}`", self.repo_root.display()); let _ = writeln!( &mut markdown, "- Iterations: `{}` recorded, `{}` warmup", self.config.iterations, self.config.warmup_iterations ); let _ = writeln!( &mut markdown, "- Reproduce: `cargo run -p agentos-execution --bin node-import-bench -- --iterations {} --warmup-iterations {}`", self.config.iterations, self.config.warmup_iterations ); let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Transport RTT"); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "| Channel | Payload (bytes) | Mean RTT (ms) | P50 | P95 |" ); let _ = writeln!(&mut markdown, "| --- | ---: | ---: | ---: | ---: |"); for transport in &self.transport_rtt { let _ = writeln!( &mut markdown, "| `{}` | {} | {} | {} | {} |", transport.channel, transport.payload_bytes, format_ms(transport.stats.mean_ms), format_ms(transport.stats.p50_ms), format_ms(transport.stats.p95_ms), ); } let _ = writeln!(&mut markdown, "## Control Matrix"); let _ = writeln!(&mut markdown); for row in self.control_matrix() { let _ = writeln!( &mut markdown, "- Workload `{}`: runtimes {}, modes {}, scenarios {}", row.workload, format_label_list(&row.runtimes), format_label_list(&row.modes), format_label_list(&row.scenario_ids), ); } let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Scenario Summary"); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "| Scenario | Workload | Runtime | Mode | Fixture | Cache | Mean wall (ms) | Mean context (ms) | Mean startup (ms) | Mean guest exec (ms) | Mean completion (ms) | Mean startup overhead (ms) |" ); let _ = writeln!( &mut markdown, "| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |" ); for scenario in &self.scenarios { let guest_execution_mean = scenario .phase_stats .guest_execution_ms .as_ref() .map(|stats| format_ms(stats.mean_ms)) .unwrap_or_else(|| String::from("n/a")); let startup_overhead_mean = scenario .startup_overhead_stats .as_ref() .map(|stats| format_ms(stats.mean_ms)) .unwrap_or_else(|| String::from("n/a")); let _ = writeln!( &mut markdown, "| `{}` | `{}` | `{}` | `{}` | {} | {} | {} | {} | {} | {} | {} | {} |", scenario.id, scenario.workload, scenario.runtime, scenario.mode, scenario.fixture, scenario.compile_cache, format_ms(scenario.wall_stats.mean_ms), format_ms(scenario.phase_stats.context_setup_ms.mean_ms), format_ms(scenario.phase_stats.startup_ms.mean_ms), guest_execution_mean, format_ms(scenario.phase_stats.completion_ms.mean_ms), startup_overhead_mean, ); } let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Stability And Resource Summary"); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "| Scenario | Wall P50 (ms) | Wall min-max (ms) | Wall stddev (ms) | Mean RSS (MiB) | Mean heap (MiB) | Mean total CPU (ms) |" ); let _ = writeln!( &mut markdown, "| --- | ---: | --- | ---: | ---: | ---: | ---: |" ); for scenario in &self.scenarios { let _ = writeln!( &mut markdown, "| `{}` | {} | {}-{} | {} | {} | {} | {} |", scenario.id, format_ms(scenario.wall_stats.p50_ms), format_ms(scenario.wall_stats.min_ms), format_ms(scenario.wall_stats.max_ms), format_ms(scenario.wall_stats.stddev_ms), scenario .resource_usage_stats .as_ref() .and_then(|stats| stats.rss_bytes.as_ref()) .map(|stats| format_mib(bytes_to_mib(stats.mean))) .unwrap_or_else(|| String::from("n/a")), scenario .resource_usage_stats .as_ref() .and_then(|stats| stats.heap_used_bytes.as_ref()) .map(|stats| format_mib(bytes_to_mib(stats.mean))) .unwrap_or_else(|| String::from("n/a")), scenario .resource_usage_stats .as_ref() .and_then(|stats| stats.cpu_total_us.as_ref()) .map(|stats| format_ms(micros_to_ms(stats.mean))) .unwrap_or_else(|| String::from("n/a")), ); } let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Ranked Hotspots"); let _ = writeln!(&mut markdown); for ranking in self.hotspot_rankings() { let _ = writeln!( &mut markdown, "### {} (`{}`, `{}`)", ranking.label, ranking.dimension, ranking.unit ); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "| Rank | Scenario | Workload | Runtime | Mode | Value |" ); let _ = writeln!(&mut markdown, "| ---: | --- | --- | --- | --- | ---: |"); for scenario in &ranking.ranked_scenarios { let _ = writeln!( &mut markdown, "| {} | `{}` | `{}` | `{}` | `{}` | {} |", scenario.rank, scenario.id, scenario.workload, scenario.runtime, scenario.mode, format_hotspot_value(ranking.unit, scenario.value), ); } if !ranking.scenarios_without_metric.is_empty() { let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "Missing metric for: {}", format_string_label_list(&ranking.scenarios_without_metric), ); } let _ = writeln!(&mut markdown); } let _ = writeln!(&mut markdown, "## Hotspot Guidance"); let _ = writeln!(&mut markdown); for line in self.guidance_lines() { let _ = writeln!(&mut markdown, "- {line}"); } if let Some(comparison) = comparison { let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Baseline Comparison"); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "- Baseline artifact: `{}`", comparison.baseline.path.display() ); let _ = writeln!( &mut markdown, "- Baseline generated at unix ms: `{}`", comparison.baseline.generated_at_unix_ms ); let _ = writeln!( &mut markdown, "- Compared scenarios: `{}`", comparison.summary.compared_scenario_count ); if let Some(improvement) = &comparison.summary.largest_wall_improvement { let _ = writeln!( &mut markdown, "- Largest wall-time improvement: `{}` at {} ({})", improvement.id, format_delta_ms(improvement.delta_ms), format_delta_pct(improvement.delta_pct), ); } if let Some(regression) = &comparison.summary.largest_wall_regression { let _ = writeln!( &mut markdown, "- Largest wall-time regression: `{}` at {} ({})", regression.id, format_delta_ms(regression.delta_ms), format_delta_pct(regression.delta_pct), ); } if !comparison.scenarios_missing_from_baseline.is_empty() { let _ = writeln!( &mut markdown, "- Scenarios missing from baseline: {}", comparison.scenarios_missing_from_baseline.join(", ") ); } if !comparison.baseline_only_scenarios.is_empty() { let _ = writeln!( &mut markdown, "- Baseline-only scenarios: {}", comparison.baseline_only_scenarios.join(", ") ); } let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "| Scenario | Wall delta (ms) | Wall delta % | Import delta (ms) | Startup delta (ms) | Context delta (ms) | Completion delta (ms) |" ); let _ = writeln!( &mut markdown, "| --- | ---: | ---: | ---: | ---: | ---: | ---: |" ); for scenario in &comparison.scenario_deltas { let import_delta = scenario .guest_import_mean_ms .as_ref() .map(|delta| format_delta_ms(delta.delta_ms)) .unwrap_or_else(|| String::from("n/a")); let startup_delta = scenario .startup_overhead_mean_ms .as_ref() .map(|delta| format_delta_ms(delta.delta_ms)) .unwrap_or_else(|| String::from("n/a")); let context_delta = scenario .phase_mean_ms .as_ref() .map(|delta| format_delta_ms(delta.context_setup_ms.delta_ms)) .unwrap_or_else(|| String::from("n/a")); let completion_delta = scenario .phase_mean_ms .as_ref() .map(|delta| format_delta_ms(delta.completion_ms.delta_ms)) .unwrap_or_else(|| String::from("n/a")); let _ = writeln!( &mut markdown, "| `{}` | {} | {} | {} | {} | {} | {} |", scenario.id, format_delta_ms(scenario.wall_mean_ms.delta_ms), format_delta_pct(scenario.wall_mean_ms.delta_pct), import_delta, startup_delta, context_delta, completion_delta, ); } } let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Raw Samples"); let _ = writeln!(&mut markdown); for scenario in &self.scenarios { let _ = writeln!(&mut markdown, "### `{}`", scenario.id); let _ = writeln!(&mut markdown, "- Workload: `{}`", scenario.workload); let _ = writeln!(&mut markdown, "- Runtime: `{}`", scenario.runtime);Lines 1–500 of 3600. Load more
use crate::{ CreateJavascriptContextRequest, JavascriptExecutionEngine, JavascriptExecutionError, StartJavascriptExecutionRequest, }; use agentos_runtime::RuntimeContext; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::env; use std::fmt; use std::fmt::Write as _; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; const BENCHMARK_MARKER_PREFIX: &str = "__AGENTOS_BENCH__:"; const LOCAL_GRAPH_MODULE_COUNT: usize = 24; const BENCHMARK_ARTIFACT_VERSION: u32 = 5; const BENCHMARK_ARTIFACT_DIR: &str = "target/benchmark-reports/node-import-bench"; const BENCHMARK_RUN_STATE_FILE: &str = "run-state.json"; const TRANSPORT_RTT_CHANNEL: &str = "execution-stdio-echo"; const TRANSPORT_RTT_PAYLOAD_BYTES: [usize; 3] = [32, 4 * 1024, 64 * 1024]; const TRANSPORT_POLL_TIMEOUT: Duration = Duration::from_secs(5); const MAX_BENCHMARK_ITERATIONS: usize = 1_000; const MAX_BENCHMARK_WARMUP_ITERATIONS: usize = 1_000; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct JavascriptBenchmarkConfig { pub iterations: usize, pub warmup_iterations: usize, } impl Default for JavascriptBenchmarkConfig { fn default() -> Self { Self { iterations: 5, warmup_iterations: 1, } } } #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct BenchmarkHost { pub node_binary: String, pub node_version: String, pub os: &'static str, pub arch: &'static str, pub logical_cpus: usize, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct BenchmarkScenarioPhases<T> { pub context_setup_ms: T, pub startup_ms: T, #[serde(skip_serializing_if = "Option::is_none", default)] pub guest_execution_ms: Option<T>, pub completion_ms: T, } #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] pub struct BenchmarkStats { pub mean_ms: f64, pub p50_ms: f64, pub p95_ms: f64, pub min_ms: f64, pub max_ms: f64, pub stddev_ms: f64, } #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] pub struct BenchmarkDistributionStats { pub mean: f64, pub p50: f64, pub p95: f64, pub min: f64, pub max: f64, pub stddev: f64, } #[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] pub struct BenchmarkResourceUsage<T> { #[serde(skip_serializing_if = "Option::is_none", default)] pub rss_bytes: Option<T>, #[serde(skip_serializing_if = "Option::is_none", default)] pub heap_used_bytes: Option<T>, #[serde(skip_serializing_if = "Option::is_none", default)] pub cpu_user_us: Option<T>, #[serde(skip_serializing_if = "Option::is_none", default)] pub cpu_system_us: Option<T>, #[serde(skip_serializing_if = "Option::is_none", default)] pub cpu_total_us: Option<T>, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BenchmarkTransportRttReport { pub channel: &'static str, pub payload_bytes: usize, pub samples_ms: Vec<f64>, pub stats: BenchmarkStats, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BenchmarkScenarioReport { pub id: &'static str, pub workload: &'static str, pub runtime: &'static str, pub mode: &'static str, pub description: &'static str, pub fixture: &'static str, pub compile_cache: &'static str, pub wall_samples_ms: Vec<f64>, pub wall_stats: BenchmarkStats, pub guest_import_samples_ms: Option<Vec<f64>>, pub guest_import_stats: Option<BenchmarkStats>, pub startup_overhead_samples_ms: Option<Vec<f64>>, pub startup_overhead_stats: Option<BenchmarkStats>, pub phase_samples_ms: BenchmarkScenarioPhases<Vec<f64>>, pub phase_stats: BenchmarkScenarioPhases<BenchmarkStats>, #[serde(skip_serializing_if = "Option::is_none", default)] pub resource_usage_samples: Option<BenchmarkResourceUsage<Vec<f64>>>, #[serde(skip_serializing_if = "Option::is_none", default)] pub resource_usage_stats: Option<BenchmarkResourceUsage<BenchmarkDistributionStats>>, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct JavascriptBenchmarkReport { pub generated_at_unix_ms: u128, pub config: JavascriptBenchmarkConfig, pub host: BenchmarkHost, pub repo_root: PathBuf, pub transport_rtt: Vec<BenchmarkTransportRttReport>, pub scenarios: Vec<BenchmarkScenarioReport>, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BenchmarkComparison { pub baseline: BenchmarkComparisonBaseline, pub summary: BenchmarkComparisonSummary, pub scenario_deltas: Vec<BenchmarkScenarioDelta>, pub scenarios_missing_from_baseline: Vec<String>, pub baseline_only_scenarios: Vec<String>, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BenchmarkComparisonBaseline { pub artifact_version: u32, pub generated_at_unix_ms: u128, pub path: PathBuf, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BenchmarkComparisonSummary { pub compared_scenario_count: usize, #[serde(skip_serializing_if = "Option::is_none")] pub largest_wall_improvement: Option<BenchmarkDeltaHighlight>, #[serde(skip_serializing_if = "Option::is_none")] pub largest_wall_regression: Option<BenchmarkDeltaHighlight>, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BenchmarkDeltaHighlight { pub id: String, pub delta_ms: f64, pub delta_pct: f64, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BenchmarkScenarioDelta { pub id: String, pub description: String, pub wall_mean_ms: BenchmarkMetricDelta, #[serde(skip_serializing_if = "Option::is_none")] pub guest_import_mean_ms: Option<BenchmarkMetricDelta>, #[serde(skip_serializing_if = "Option::is_none")] pub startup_overhead_mean_ms: Option<BenchmarkMetricDelta>, #[serde(skip_serializing_if = "Option::is_none")] pub phase_mean_ms: Option<BenchmarkScenarioPhases<BenchmarkMetricDelta>>, } #[derive(Debug, Clone, PartialEq, Serialize)] pub struct BenchmarkMetricDelta { pub baseline_ms: f64, pub current_ms: f64, pub delta_ms: f64, pub delta_pct: f64, } impl JavascriptBenchmarkReport { pub fn render_markdown(&self) -> String { self.render_markdown_with_comparison(None) } pub fn render_markdown_with_comparison( &self, comparison: Option<&BenchmarkComparison>, ) -> String { let mut markdown = String::new(); let _ = writeln!(&mut markdown, "# secure-exec Node Import Benchmark"); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "- Generated at unix ms: `{}`", self.generated_at_unix_ms ); let _ = writeln!(&mut markdown, "- Node binary: `{}`", self.host.node_binary); let _ = writeln!( &mut markdown, "- Node version: `{}`", self.host.node_version.trim() ); let _ = writeln!( &mut markdown, "- Host: `{}` / `{}` / `{}` logical CPUs", self.host.os, self.host.arch, self.host.logical_cpus ); let _ = writeln!(&mut markdown, "- Repo root: `{}`", self.repo_root.display()); let _ = writeln!( &mut markdown, "- Iterations: `{}` recorded, `{}` warmup", self.config.iterations, self.config.warmup_iterations ); let _ = writeln!( &mut markdown, "- Reproduce: `cargo run -p agentos-execution --bin node-import-bench -- --iterations {} --warmup-iterations {}`", self.config.iterations, self.config.warmup_iterations ); let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Transport RTT"); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "| Channel | Payload (bytes) | Mean RTT (ms) | P50 | P95 |" ); let _ = writeln!(&mut markdown, "| --- | ---: | ---: | ---: | ---: |"); for transport in &self.transport_rtt { let _ = writeln!( &mut markdown, "| `{}` | {} | {} | {} | {} |", transport.channel, transport.payload_bytes, format_ms(transport.stats.mean_ms), format_ms(transport.stats.p50_ms), format_ms(transport.stats.p95_ms), ); } let _ = writeln!(&mut markdown, "## Control Matrix"); let _ = writeln!(&mut markdown); for row in self.control_matrix() { let _ = writeln!( &mut markdown, "- Workload `{}`: runtimes {}, modes {}, scenarios {}", row.workload, format_label_list(&row.runtimes), format_label_list(&row.modes), format_label_list(&row.scenario_ids), ); } let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Scenario Summary"); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "| Scenario | Workload | Runtime | Mode | Fixture | Cache | Mean wall (ms) | Mean context (ms) | Mean startup (ms) | Mean guest exec (ms) | Mean completion (ms) | Mean startup overhead (ms) |" ); let _ = writeln!( &mut markdown, "| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |" ); for scenario in &self.scenarios { let guest_execution_mean = scenario .phase_stats .guest_execution_ms .as_ref() .map(|stats| format_ms(stats.mean_ms)) .unwrap_or_else(|| String::from("n/a")); let startup_overhead_mean = scenario .startup_overhead_stats .as_ref() .map(|stats| format_ms(stats.mean_ms)) .unwrap_or_else(|| String::from("n/a")); let _ = writeln!( &mut markdown, "| `{}` | `{}` | `{}` | `{}` | {} | {} | {} | {} | {} | {} | {} | {} |", scenario.id, scenario.workload, scenario.runtime, scenario.mode, scenario.fixture, scenario.compile_cache, format_ms(scenario.wall_stats.mean_ms), format_ms(scenario.phase_stats.context_setup_ms.mean_ms), format_ms(scenario.phase_stats.startup_ms.mean_ms), guest_execution_mean, format_ms(scenario.phase_stats.completion_ms.mean_ms), startup_overhead_mean, ); } let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Stability And Resource Summary"); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "| Scenario | Wall P50 (ms) | Wall min-max (ms) | Wall stddev (ms) | Mean RSS (MiB) | Mean heap (MiB) | Mean total CPU (ms) |" ); let _ = writeln!( &mut markdown, "| --- | ---: | --- | ---: | ---: | ---: | ---: |" ); for scenario in &self.scenarios { let _ = writeln!( &mut markdown, "| `{}` | {} | {}-{} | {} | {} | {} | {} |", scenario.id, format_ms(scenario.wall_stats.p50_ms), format_ms(scenario.wall_stats.min_ms), format_ms(scenario.wall_stats.max_ms), format_ms(scenario.wall_stats.stddev_ms), scenario .resource_usage_stats .as_ref() .and_then(|stats| stats.rss_bytes.as_ref()) .map(|stats| format_mib(bytes_to_mib(stats.mean))) .unwrap_or_else(|| String::from("n/a")), scenario .resource_usage_stats .as_ref() .and_then(|stats| stats.heap_used_bytes.as_ref()) .map(|stats| format_mib(bytes_to_mib(stats.mean))) .unwrap_or_else(|| String::from("n/a")), scenario .resource_usage_stats .as_ref() .and_then(|stats| stats.cpu_total_us.as_ref()) .map(|stats| format_ms(micros_to_ms(stats.mean))) .unwrap_or_else(|| String::from("n/a")), ); } let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Ranked Hotspots"); let _ = writeln!(&mut markdown); for ranking in self.hotspot_rankings() { let _ = writeln!( &mut markdown, "### {} (`{}`, `{}`)", ranking.label, ranking.dimension, ranking.unit ); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "| Rank | Scenario | Workload | Runtime | Mode | Value |" ); let _ = writeln!(&mut markdown, "| ---: | --- | --- | --- | --- | ---: |"); for scenario in &ranking.ranked_scenarios { let _ = writeln!( &mut markdown, "| {} | `{}` | `{}` | `{}` | `{}` | {} |", scenario.rank, scenario.id, scenario.workload, scenario.runtime, scenario.mode, format_hotspot_value(ranking.unit, scenario.value), ); } if !ranking.scenarios_without_metric.is_empty() { let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "Missing metric for: {}", format_string_label_list(&ranking.scenarios_without_metric), ); } let _ = writeln!(&mut markdown); } let _ = writeln!(&mut markdown, "## Hotspot Guidance"); let _ = writeln!(&mut markdown); for line in self.guidance_lines() { let _ = writeln!(&mut markdown, "- {line}"); } if let Some(comparison) = comparison { let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Baseline Comparison"); let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "- Baseline artifact: `{}`", comparison.baseline.path.display() ); let _ = writeln!( &mut markdown, "- Baseline generated at unix ms: `{}`", comparison.baseline.generated_at_unix_ms ); let _ = writeln!( &mut markdown, "- Compared scenarios: `{}`", comparison.summary.compared_scenario_count ); if let Some(improvement) = &comparison.summary.largest_wall_improvement { let _ = writeln!( &mut markdown, "- Largest wall-time improvement: `{}` at {} ({})", improvement.id, format_delta_ms(improvement.delta_ms), format_delta_pct(improvement.delta_pct), ); } if let Some(regression) = &comparison.summary.largest_wall_regression { let _ = writeln!( &mut markdown, "- Largest wall-time regression: `{}` at {} ({})", regression.id, format_delta_ms(regression.delta_ms), format_delta_pct(regression.delta_pct), ); } if !comparison.scenarios_missing_from_baseline.is_empty() { let _ = writeln!( &mut markdown, "- Scenarios missing from baseline: {}", comparison.scenarios_missing_from_baseline.join(", ") ); } if !comparison.baseline_only_scenarios.is_empty() { let _ = writeln!( &mut markdown, "- Baseline-only scenarios: {}", comparison.baseline_only_scenarios.join(", ") ); } let _ = writeln!(&mut markdown); let _ = writeln!( &mut markdown, "| Scenario | Wall delta (ms) | Wall delta % | Import delta (ms) | Startup delta (ms) | Context delta (ms) | Completion delta (ms) |" ); let _ = writeln!( &mut markdown, "| --- | ---: | ---: | ---: | ---: | ---: | ---: |" ); for scenario in &comparison.scenario_deltas { let import_delta = scenario .guest_import_mean_ms .as_ref() .map(|delta| format_delta_ms(delta.delta_ms)) .unwrap_or_else(|| String::from("n/a")); let startup_delta = scenario .startup_overhead_mean_ms .as_ref() .map(|delta| format_delta_ms(delta.delta_ms)) .unwrap_or_else(|| String::from("n/a")); let context_delta = scenario .phase_mean_ms .as_ref() .map(|delta| format_delta_ms(delta.context_setup_ms.delta_ms)) .unwrap_or_else(|| String::from("n/a")); let completion_delta = scenario .phase_mean_ms .as_ref() .map(|delta| format_delta_ms(delta.completion_ms.delta_ms)) .unwrap_or_else(|| String::from("n/a")); let _ = writeln!( &mut markdown, "| `{}` | {} | {} | {} | {} | {} | {} |", scenario.id, format_delta_ms(scenario.wall_mean_ms.delta_ms), format_delta_pct(scenario.wall_mean_ms.delta_pct), import_delta, startup_delta, context_delta, completion_delta, ); } } let _ = writeln!(&mut markdown); let _ = writeln!(&mut markdown, "## Raw Samples"); let _ = writeln!(&mut markdown); for scenario in &self.scenarios { let _ = writeln!(&mut markdown, "### `{}`", scenario.id); let _ = writeln!(&mut markdown, "- Workload: `{}`", scenario.workload); let _ = writeln!(&mut markdown, "- Runtime: `{}`", scenario.runtime);