| @@ -0,0 +1,429 @@ |
| | 1 | const fs = require("node:fs"); |
| | 2 | const path = require("node:path"); |
| | 3 | |
| | 4 | const ARTIFACT_NAME = "thread-transfer-results"; |
| | 5 | const RESULT_FILE = "thread-transfer-result.json"; |
| | 6 | const COMMENT_MARKER = "<!-- t3-thread-transfer-report -->"; |
| | 7 | const PROVIDERS = ["codex", "claudeAgent"]; |
| | 8 | const OBSERVED_KEYS = [ |
| | 9 | "totalWireBytes", |
| | 10 | "threadSnapshotWireBytes", |
| | 11 | "threadSnapshotDecodedBytes", |
| | 12 | "measuredTurnWebSocketWireBytes", |
| | 13 | "measuredTurnWebSocketDecodedBytes", |
| | 14 | "measuredTurnWebSocketMessages", |
| | 15 | ]; |
| | 16 | const CEILING_KEYS = [ |
| | 17 | "totalWireBytes", |
| | 18 | "threadSnapshotWireBytes", |
| | 19 | "measuredTurnWebSocketWireBytes", |
| | 20 | "measuredTurnWebSocketDecodedBytes", |
| | 21 | "measuredTurnWebSocketMessages", |
| | 22 | ]; |
| | 23 | const SCENARIO_KEYS = [ |
| | 24 | "id", |
| | 25 | "historyTurns", |
| | 26 | "historyCommandToolsPerTurn", |
| | 27 | "historyMcpResultBytes", |
| | 28 | "measuredCommandTools", |
| | 29 | "measuredMcpResultBytes", |
| | 30 | ]; |
| | 31 | |
| | 32 | function resultShaMarker(sha) { |
| | 33 | return `<!-- t3-thread-transfer-result-sha:${sha} -->`; |
| | 34 | } |
| | 35 | |
| | 36 | function assertObject(value, label) { |
| | 37 | if (value === null || typeof value !== "object" || Array.isArray(value)) { |
| | 38 | throw new Error(`${label} must be an object`); |
| | 39 | } |
| | 40 | } |
| | 41 | |
| | 42 | function assertExactKeys(value, expected, label) { |
| | 43 | assertObject(value, label); |
| | 44 | const actual = Object.keys(value).sort(); |
| | 45 | const wanted = [...expected].sort(); |
| | 46 | if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index])) { |
| | 47 | throw new Error(`${label} has unexpected fields`); |
| | 48 | } |
| | 49 | } |
| | 50 | |
| | 51 | function assertMetric(value, label) { |
| | 52 | if (!Number.isSafeInteger(value) || value < 0 || value > 1_000_000_000) { |
| | 53 | throw new Error(`${label} must be a non-negative safe integer below 1,000,000,000`); |
| | 54 | } |
| | 55 | } |
| | 56 | |
| | 57 | function validateResult(value) { |
| | 58 | assertExactKeys(value, ["schemaVersion", "scenario", "providers"], "result"); |
| | 59 | if (value.schemaVersion !== 1) { |
| | 60 | throw new Error("result.schemaVersion must be 1"); |
| | 61 | } |
| | 62 | |
| | 63 | assertExactKeys(value.scenario, SCENARIO_KEYS, "result.scenario"); |
| | 64 | if (value.scenario.id !== "thread-transfer-v1") { |
| | 65 | throw new Error("result.scenario.id is not supported"); |
| | 66 | } |
| | 67 | for (const key of SCENARIO_KEYS.slice(1)) { |
| | 68 | assertMetric(value.scenario[key], `result.scenario.${key}`); |
| | 69 | } |
| | 70 | |
| | 71 | assertExactKeys(value.providers, PROVIDERS, "result.providers"); |
| | 72 | for (const provider of PROVIDERS) { |
| | 73 | const entry = value.providers[provider]; |
| | 74 | assertExactKeys(entry, ["observed", "ceiling"], `result.providers.${provider}`); |
| | 75 | assertExactKeys(entry.observed, OBSERVED_KEYS, `result.providers.${provider}.observed`); |
| | 76 | assertExactKeys(entry.ceiling, CEILING_KEYS, `result.providers.${provider}.ceiling`); |
| | 77 | for (const key of OBSERVED_KEYS) { |
| | 78 | assertMetric(entry.observed[key], `result.providers.${provider}.observed.${key}`); |
| | 79 | } |
| | 80 | for (const key of CEILING_KEYS) { |
| | 81 | assertMetric(entry.ceiling[key], `result.providers.${provider}.ceiling.${key}`); |
| | 82 | } |
| | 83 | } |
| | 84 | |
| | 85 | return value; |
| | 86 | } |
| | 87 | |
| | 88 | function readResult(directory) { |
| | 89 | if (!directory) return undefined; |
| | 90 | const file = path.join(directory, RESULT_FILE); |
| | 91 | if (!fs.existsSync(file)) return undefined; |
| | 92 | const stat = fs.lstatSync(file); |
| | 93 | if (!stat.isFile() || stat.size > 64 * 1_024) { |
| | 94 | throw new Error("thread transfer result must be a regular file smaller than 64 KiB"); |
| | 95 | } |
| | 96 | return validateResult(JSON.parse(fs.readFileSync(file, "utf8"))); |
| | 97 | } |
| | 98 | |
| | 99 | function formatBytes(bytes) { |
| | 100 | if (bytes < 1_024) return `${bytes} B`; |
| | 101 | if (bytes >= 1_024 * 1_024) return `${(bytes / 1_024 / 1_024).toFixed(2)} MiB`; |
| | 102 | return `${(bytes / 1_024).toFixed(1)} KiB`; |
| | 103 | } |
| | 104 | |
| | 105 | function formatValue(value, kind) { |
| | 106 | return kind === "messages" ? value.toLocaleString("en-US") : formatBytes(value); |
| | 107 | } |
| | 108 | |
| | 109 | function formatImpact(current, baseline, kind) { |
| | 110 | if (baseline === undefined) return "—"; |
| | 111 | const delta = current - baseline; |
| | 112 | const prefix = delta > 0 ? "+" : delta < 0 ? "−" : ""; |
| | 113 | const magnitude = formatValue(Math.abs(delta), kind); |
| | 114 | const percent = |
| | 115 | baseline === 0 ? "" : ` (${prefix}${Math.abs((delta / baseline) * 100).toFixed(1)}%)`; |
| | 116 | return `${prefix}${magnitude}${percent}`; |
| | 117 | } |
| | 118 | |
| | 119 | function sameScenario(left, right) { |
| | 120 | return SCENARIO_KEYS.every((key) => left[key] === right[key]); |
| | 121 | } |
| | 122 | |
| | 123 | const METRICS = [ |
| | 124 | { key: "totalWireBytes", label: "Total thread wire", kind: "bytes" }, |
| | 125 | { key: "threadSnapshotWireBytes", label: "Thread snapshot wire", kind: "bytes" }, |
| | 126 | { |
| | 127 | key: "measuredTurnWebSocketWireBytes", |
| | 128 | label: "Live turn WebSocket wire", |
| | 129 | kind: "bytes", |
| | 130 | }, |
| | 131 | { |
| | 132 | key: "measuredTurnWebSocketDecodedBytes", |
| | 133 | label: "Live turn WebSocket decoded", |
| | 134 | kind: "bytes", |
| | 135 | }, |
| | 136 | { key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" }, |
| | 137 | ]; |
| | 138 | |
| | 139 | function renderComment(input) { |
| | 140 | const current = input.current; |
| | 141 | const baseline = input.baseline; |
| | 142 | const comparable = baseline !== undefined && sameScenario(current.scenario, baseline.scenario); |
| | 143 | const rows = []; |
| | 144 | const ceilingChanges = []; |
| | 145 | let failed = false; |
| | 146 | |
| | 147 | for (const provider of PROVIDERS) { |
| | 148 | for (const metric of METRICS) { |
| | 149 | const observed = current.providers[provider].observed[metric.key]; |
| | 150 | const ceiling = current.providers[provider].ceiling[metric.key]; |
| | 151 | const baselineObserved = comparable |
| | 152 | ? baseline.providers[provider].observed[metric.key] |
| | 153 | : undefined; |
| | 154 | const pass = observed <= ceiling; |
| | 155 | failed ||= !pass; |
| | 156 | rows.push( |
| | 157 | `| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`, |
| | 158 | ); |
| | 159 | |
| | 160 | if (baseline && baseline.providers[provider].ceiling[metric.key] !== ceiling) { |
| | 161 | ceilingChanges.push( |
| | 162 | `- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`, |
| | 163 | ); |
| | 164 | } |
| | 165 | } |
| | 166 | } |
| | 167 | |
| | 168 | const baselineLink = input.baselineRun |
| | 169 | ? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})` |
| | 170 | : "unavailable"; |
| | 171 | const currentLink = `[\`${input.currentRun.sha.slice(0, 7)}\`](${input.currentRun.url})`; |
| | 172 | const notices = []; |
| | 173 | if (!baseline) { |
| | 174 | notices.push( |
| | 175 | "> ℹ️ No successful `main` baseline artifact is available yet. This run establishes the initial measurement.", |
| | 176 | ); |
| | 177 | } else if (!comparable) { |
| | 178 | notices.push( |
| | 179 | "> ⚠️ The thread fixture changed, so impact percentages are not directly comparable to the `main` baseline.", |
| | 180 | ); |
| | 181 | } else if (!input.baselineRun.matchesBase) { |
| | 182 | notices.push( |
| | 183 | "> ℹ️ The exact PR base did not have a successful artifact. Baseline uses the latest successful `main` measurement shown below.", |
| | 184 | ); |
| | 185 | } |
| | 186 | if (ceilingChanges.length > 0) { |
| | 187 | notices.push( |
| | 188 | `> ⚠️ **This PR changes transfer ceilings:**\n>\n${ceilingChanges.map((line) => `> ${line}`).join("\n")}`, |
| | 189 | ); |
| | 190 | } |
| | 191 | |
| | 192 | return [ |
| | 193 | COMMENT_MARKER, |
| | 194 | resultShaMarker(input.currentRun.sha), |
| | 195 | "## Thread transfer impact", |
| | 196 | "", |
| | 197 | failed |
| | 198 | ? "❌ One or more thread transfer ceilings were exceeded." |
| | 199 | : "✅ Thread transfer remains within every enforced ceiling.", |
| | 200 | ...(notices.length > 0 ? ["", ...notices] : []), |
| | 201 | "", |
| | 202 | "| Provider | Metric | Main baseline | This PR | Impact | PR ceiling | |", |
| | 203 | "| --- | --- | ---: | ---: | ---: | ---: | --- |", |
| | 204 | ...rows, |
| | 205 | "", |
| | 206 | `Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`, |
| | 207 | "", |
| | 208 | "<details>", |
| | 209 | "<summary>Scenario and decoded snapshot size</summary>", |
| | 210 | "", |
| | 211 | `${current.scenario.historyTurns} historical turns, ${current.scenario.historyCommandToolsPerTurn} command tools per turn, ${formatBytes(current.scenario.historyMcpResultBytes)} retained MCP result per historical turn, and a ${formatBytes(current.scenario.measuredMcpResultBytes)} retained result in the measured turn.`, |
| | 212 | "", |
| | 213 | ...PROVIDERS.map( |
| | 214 | (provider) => |
| | 215 | `- ${provider === "codex" ? "Codex" : "Claude"} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`, |
| | 216 | ), |
| | 217 | "", |
| | 218 | "</details>", |
| | 219 | "", |
| | 220 | "_Updated in place by a trusted workflow. PR artifacts are strictly validated and never executed._", |
| | 221 | ].join("\n"); |
| | 222 | } |
| | 223 | |
| | 224 | async function artifactsForRun(github, owner, repo, runId) { |
| | 225 | return github.paginate(github.rest.actions.listWorkflowRunArtifacts, { |
| | 226 | owner, |
| | 227 | repo, |
| | 228 | run_id: runId, |
| | 229 | per_page: 100, |
| | 230 | }); |
| | 231 | } |
| | 232 | |
| | 233 | function findResultArtifact(artifacts) { |
| | 234 | return artifacts.find((artifact) => artifact.name === ARTIFACT_NAME && !artifact.expired); |
| | 235 | } |
| | 236 | |
| | 237 | async function resolve({ github, context, core }) { |
| | 238 | const source = context.payload.workflow_run; |
| | 239 | const { owner, repo } = context.repo; |
| | 240 | if (source.event !== "pull_request") { |
| | 241 | core.setOutput("publish", "false"); |
| | 242 | return; |
| | 243 | } |
| | 244 | |
| | 245 | let pullNumber = source.pull_requests?.[0]?.number; |
| | 246 | if (!pullNumber) { |
| | 247 | const associated = await github.paginate( |
| | 248 | github.rest.repos.listPullRequestsAssociatedWithCommit, |
| | 249 | { owner, repo, commit_sha: source.head_sha, per_page: 100 }, |
| | 250 | ); |
| | 251 | const matchingPulls = associated.filter( |
| | 252 | (pull) => |
| | 253 | pull.state === "open" && |
| | 254 | pull.head.sha === source.head_sha && |
| | 255 | pull.head.ref === source.head_branch, |
| | 256 | ); |
| | 257 | if (matchingPulls.length !== 1) { |
| | 258 | core.info( |
| | 259 | `Expected one open pull request for ${source.head_repository?.full_name ?? "unknown repository"}:${source.head_branch ?? "unknown branch"} at ${source.head_sha}; found ${matchingPulls.length}.`, |
| | 260 | ); |
| | 261 | core.setOutput("publish", "false"); |
| | 262 | return; |
| | 263 | } |
| | 264 | pullNumber = matchingPulls[0].number; |
| | 265 | } |
| | 266 | if (!pullNumber) { |
| | 267 | core.info("No open pull request is associated with the completed CI run."); |
| | 268 | core.setOutput("publish", "false"); |
| | 269 | return; |
| | 270 | } |
| | 271 | |
| | 272 | const { data: pull } = await github.rest.pulls.get({ owner, repo, pull_number: pullNumber }); |
| | 273 | if (pull.head.sha !== source.head_sha) { |
| | 274 | core.info(`Skipping stale CI result ${source.head_sha}; PR head is ${pull.head.sha}.`); |
| | 275 | core.setOutput("publish", "false"); |
| | 276 | return; |
| | 277 | } |
| | 278 | |
| | 279 | const sourceArtifacts = await artifactsForRun(github, owner, repo, source.id); |
| | 280 | const sourceArtifact = findResultArtifact(sourceArtifacts); |
| | 281 | const workflowRuns = await github.paginate(github.rest.actions.listWorkflowRuns, { |
| | 282 | owner, |
| | 283 | repo, |
| | 284 | workflow_id: source.workflow_id, |
| | 285 | branch: pull.base.ref, |
| | 286 | event: "push", |
| | 287 | status: "success", |
| | 288 | per_page: 100, |
| | 289 | }); |
| | 290 | const orderedRuns = [ |
| | 291 | ...workflowRuns.filter((run) => run.head_sha === pull.base.sha), |
| | 292 | ...workflowRuns.filter((run) => run.head_sha !== pull.base.sha), |
| | 293 | ].slice(0, 20); |
| | 294 | |
| | 295 | let baselineRun; |
| | 296 | for (const run of orderedRuns) { |
| | 297 | const artifacts = await artifactsForRun(github, owner, repo, run.id); |
| | 298 | if (findResultArtifact(artifacts)) { |
| | 299 | baselineRun = run; |
| | 300 | break; |
| | 301 | } |
| | 302 | } |
| | 303 | |
| | 304 | core.setOutput("publish", "true"); |
| | 305 | core.setOutput("pull_number", String(pullNumber)); |
| | 306 | core.setOutput("pr_artifact", sourceArtifact ? "true" : "false"); |
| | 307 | core.setOutput("pr_run_id", String(source.id)); |
| | 308 | core.setOutput("pr_sha", source.head_sha); |
| | 309 | core.setOutput("pr_conclusion", source.conclusion ?? "unknown"); |
| | 310 | core.setOutput("baseline_artifact", baselineRun ? "true" : "false"); |
| | 311 | core.setOutput("baseline_run_id", baselineRun ? String(baselineRun.id) : ""); |
| | 312 | core.setOutput("baseline_sha", baselineRun?.head_sha ?? ""); |
| | 313 | core.setOutput( |
| | 314 | "baseline_matches_base", |
| | 315 | baselineRun?.head_sha === pull.base.sha ? "true" : "false", |
| | 316 | ); |
| | 317 | } |
| | 318 | |
| | 319 | async function upsertComment(github, context, pullNumber, body, options = {}) { |
| | 320 | const { owner, repo } = context.repo; |
| | 321 | const comments = await github.paginate(github.rest.issues.listComments, { |
| | 322 | owner, |
| | 323 | repo, |
| | 324 | issue_number: pullNumber, |
| | 325 | per_page: 100, |
| | 326 | }); |
| | 327 | const existing = comments.find( |
| | 328 | (comment) => |
| | 329 | comment.user?.login === "github-actions[bot]" && comment.body?.includes(COMMENT_MARKER), |
| | 330 | ); |
| | 331 | if ( |
| | 332 | options.preserveResultSha && |
| | 333 | existing?.body?.includes(resultShaMarker(options.preserveResultSha)) |
| | 334 | ) { |
| | 335 | return; |
| | 336 | } |
| | 337 | if (existing) { |
| | 338 | await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); |
| | 339 | } else { |
| | 340 | await github.rest.issues.createComment({ owner, repo, issue_number: pullNumber, body }); |
| | 341 | } |
| | 342 | } |
| | 343 | |
| | 344 | async function upsertCommentForCurrentHead( |
| | 345 | github, |
| | 346 | context, |
| | 347 | core, |
| | 348 | pullNumber, |
| | 349 | expectedSha, |
| | 350 | body, |
| | 351 | options, |
| | 352 | ) { |
| | 353 | const { owner, repo } = context.repo; |
| | 354 | const { data: pull } = await github.rest.pulls.get({ |
| | 355 | owner, |
| | 356 | repo, |
| | 357 | pull_number: pullNumber, |
| | 358 | }); |
| | 359 | if (pull.head.sha !== expectedSha) { |
| | 360 | core.info(`Skipping stale CI result ${expectedSha}; PR head is ${pull.head.sha}.`); |
| | 361 | return false; |
| | 362 | } |
| | 363 | |
| | 364 | await upsertComment(github, context, pullNumber, body, options); |
| | 365 | return true; |
| | 366 | } |
| | 367 | |
| | 368 | async function publish({ github, context, core }) { |
| | 369 | const pullNumber = Number(process.env.PR_NUMBER); |
| | 370 | if (!Number.isSafeInteger(pullNumber) || pullNumber <= 0) { |
| | 371 | throw new Error("PR_NUMBER is invalid"); |
| | 372 | } |
| | 373 | |
| | 374 | const current = readResult(process.env.PR_RESULT_DIR); |
| | 375 | const currentRun = { |
| | 376 | sha: process.env.PR_SHA, |
| | 377 | conclusion: process.env.PR_CONCLUSION, |
| | 378 | url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.PR_RUN_ID}`, |
| | 379 | }; |
| | 380 | if (!current) { |
| | 381 | await upsertCommentForCurrentHead( |
| | 382 | github, |
| | 383 | context, |
| | 384 | core, |
| | 385 | pullNumber, |
| | 386 | currentRun.sha, |
| | 387 | [ |
| | 388 | COMMENT_MARKER, |
| | 389 | "## Thread transfer impact", |
| | 390 | "", |
| | 391 | `⚠️ The latest [CI run](${currentRun.url}) did not produce a thread transfer result for \`${currentRun.sha.slice(0, 7)}\`.`, |
| | 392 | "", |
| | 393 | "_This comment will update automatically after the next completed run._", |
| | 394 | ].join("\n"), |
| | 395 | { preserveResultSha: currentRun.sha }, |
| | 396 | ); |
| | 397 | return; |
| | 398 | } |
| | 399 | |
| | 400 | const baseline = readResult(process.env.BASELINE_RESULT_DIR); |
| | 401 | const baselineRun = baseline |
| | 402 | ? { |
| | 403 | sha: process.env.BASELINE_SHA, |
| | 404 | matchesBase: process.env.BASELINE_MATCHES_BASE === "true", |
| | 405 | url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`, |
| | 406 | } |
| | 407 | : undefined; |
| | 408 | const body = renderComment({ current, baseline, currentRun, baselineRun }); |
| | 409 | const published = await upsertCommentForCurrentHead( |
| | 410 | github, |
| | 411 | context, |
| | 412 | core, |
| | 413 | pullNumber, |
| | 414 | currentRun.sha, |
| | 415 | body, |
| | 416 | ); |
| | 417 | if (published) { |
| | 418 | core.info(`Updated thread transfer report on PR #${pullNumber}.`); |
| | 419 | } |
| | 420 | } |
| | 421 | |
| | 422 | module.exports = { |
| | 423 | publish, |
| | 424 | readResult, |
| | 425 | renderComment, |
| | 426 | resolve, |
| | 427 | upsertCommentForCurrentHead, |
| | 428 | validateResult, |
| | 429 | }; |