History

test(server): catch client transfer regressions in CI (#5350)

Theo Browne committed 3 months ago

Commit ddfe45c1 parent 9547cf212 files · +1973 −42Browse files

Showing the changes to .github (4 of 12 files). Show all

Unified · Split

4 files
  1. .github/scripts/thread-transfer-report.cjs +429 −0
  2. .github/scripts/thread-transfer-report.test.cjs +292 −0
  3. .github/workflows/ci.yml +21 −0
  4. .github/workflows/thread-transfer-report.yml +75 −0
.github/scripts/thread-transfer-report.cjs+429−0
@@ -0,0 +1,429 @@
1+const fs = require("node:fs");
2+const path = require("node:path");
3+
4+const ARTIFACT_NAME = "thread-transfer-results";
5+const RESULT_FILE = "thread-transfer-result.json";
6+const COMMENT_MARKER = "<!-- t3-thread-transfer-report -->";
7+const PROVIDERS = ["codex", "claudeAgent"];
8+const OBSERVED_KEYS = [
9+ "totalWireBytes",
10+ "threadSnapshotWireBytes",
11+ "threadSnapshotDecodedBytes",
12+ "measuredTurnWebSocketWireBytes",
13+ "measuredTurnWebSocketDecodedBytes",
14+ "measuredTurnWebSocketMessages",
15+];
16+const CEILING_KEYS = [
17+ "totalWireBytes",
18+ "threadSnapshotWireBytes",
19+ "measuredTurnWebSocketWireBytes",
20+ "measuredTurnWebSocketDecodedBytes",
21+ "measuredTurnWebSocketMessages",
22+];
23+const SCENARIO_KEYS = [
24+ "id",
25+ "historyTurns",
26+ "historyCommandToolsPerTurn",
27+ "historyMcpResultBytes",
28+ "measuredCommandTools",
29+ "measuredMcpResultBytes",
30+];
31+
32+function resultShaMarker(sha) {
33+ return `<!-- t3-thread-transfer-result-sha:${sha} -->`;
34+}
35+
36+function assertObject(value, label) {
37+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
38+ throw new Error(`${label} must be an object`);
39+ }
40+}
41+
42+function assertExactKeys(value, expected, label) {
43+ assertObject(value, label);
44+ const actual = Object.keys(value).sort();
45+ const wanted = [...expected].sort();
46+ if (actual.length !== wanted.length || actual.some((key, index) => key !== wanted[index])) {
47+ throw new Error(`${label} has unexpected fields`);
48+ }
49+}
50+
51+function assertMetric(value, label) {
52+ if (!Number.isSafeInteger(value) || value < 0 || value > 1_000_000_000) {
53+ throw new Error(`${label} must be a non-negative safe integer below 1,000,000,000`);
54+ }
55+}
56+
57+function validateResult(value) {
58+ assertExactKeys(value, ["schemaVersion", "scenario", "providers"], "result");
59+ if (value.schemaVersion !== 1) {
60+ throw new Error("result.schemaVersion must be 1");
61+ }
62+
63+ assertExactKeys(value.scenario, SCENARIO_KEYS, "result.scenario");
64+ if (value.scenario.id !== "thread-transfer-v1") {
65+ throw new Error("result.scenario.id is not supported");
66+ }
67+ for (const key of SCENARIO_KEYS.slice(1)) {
68+ assertMetric(value.scenario[key], `result.scenario.${key}`);
69+ }
70+
71+ assertExactKeys(value.providers, PROVIDERS, "result.providers");
72+ for (const provider of PROVIDERS) {
73+ const entry = value.providers[provider];
74+ assertExactKeys(entry, ["observed", "ceiling"], `result.providers.${provider}`);
75+ assertExactKeys(entry.observed, OBSERVED_KEYS, `result.providers.${provider}.observed`);
76+ assertExactKeys(entry.ceiling, CEILING_KEYS, `result.providers.${provider}.ceiling`);
77+ for (const key of OBSERVED_KEYS) {
78+ assertMetric(entry.observed[key], `result.providers.${provider}.observed.${key}`);
79+ }
80+ for (const key of CEILING_KEYS) {
81+ assertMetric(entry.ceiling[key], `result.providers.${provider}.ceiling.${key}`);
82+ }
83+ }
84+
85+ return value;
86+}
87+
88+function readResult(directory) {
89+ if (!directory) return undefined;
90+ const file = path.join(directory, RESULT_FILE);
91+ if (!fs.existsSync(file)) return undefined;
92+ const stat = fs.lstatSync(file);
93+ if (!stat.isFile() || stat.size > 64 * 1_024) {
94+ throw new Error("thread transfer result must be a regular file smaller than 64 KiB");
95+ }
96+ return validateResult(JSON.parse(fs.readFileSync(file, "utf8")));
97+}
98+
99+function formatBytes(bytes) {
100+ if (bytes < 1_024) return `${bytes} B`;
101+ if (bytes >= 1_024 * 1_024) return `${(bytes / 1_024 / 1_024).toFixed(2)} MiB`;
102+ return `${(bytes / 1_024).toFixed(1)} KiB`;
103+}
104+
105+function formatValue(value, kind) {
106+ return kind === "messages" ? value.toLocaleString("en-US") : formatBytes(value);
107+}
108+
109+function formatImpact(current, baseline, kind) {
110+ if (baseline === undefined) return "—";
111+ const delta = current - baseline;
112+ const prefix = delta > 0 ? "+" : delta < 0 ? "−" : "";
113+ const magnitude = formatValue(Math.abs(delta), kind);
114+ const percent =
115+ baseline === 0 ? "" : ` (${prefix}${Math.abs((delta / baseline) * 100).toFixed(1)}%)`;
116+ return `${prefix}${magnitude}${percent}`;
117+}
118+
119+function sameScenario(left, right) {
120+ return SCENARIO_KEYS.every((key) => left[key] === right[key]);
121+}
122+
123+const METRICS = [
124+ { key: "totalWireBytes", label: "Total thread wire", kind: "bytes" },
125+ { key: "threadSnapshotWireBytes", label: "Thread snapshot wire", kind: "bytes" },
126+ {
127+ key: "measuredTurnWebSocketWireBytes",
128+ label: "Live turn WebSocket wire",
129+ kind: "bytes",
130+ },
131+ {
132+ key: "measuredTurnWebSocketDecodedBytes",
133+ label: "Live turn WebSocket decoded",
134+ kind: "bytes",
135+ },
136+ { key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" },
137+];
138+
139+function renderComment(input) {
140+ const current = input.current;
141+ const baseline = input.baseline;
142+ const comparable = baseline !== undefined && sameScenario(current.scenario, baseline.scenario);
143+ const rows = [];
144+ const ceilingChanges = [];
145+ let failed = false;
146+
147+ for (const provider of PROVIDERS) {
148+ for (const metric of METRICS) {
149+ const observed = current.providers[provider].observed[metric.key];
150+ const ceiling = current.providers[provider].ceiling[metric.key];
151+ const baselineObserved = comparable
152+ ? baseline.providers[provider].observed[metric.key]
153+ : undefined;
154+ const pass = observed <= ceiling;
155+ failed ||= !pass;
156+ rows.push(
157+ `| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
158+ );
159+
160+ if (baseline && baseline.providers[provider].ceiling[metric.key] !== ceiling) {
161+ ceilingChanges.push(
162+ `- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
163+ );
164+ }
165+ }
166+ }
167+
168+ const baselineLink = input.baselineRun
169+ ? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})`
170+ : "unavailable";
171+ const currentLink = `[\`${input.currentRun.sha.slice(0, 7)}\`](${input.currentRun.url})`;
172+ const notices = [];
173+ if (!baseline) {
174+ notices.push(
175+ "> ℹ️ No successful `main` baseline artifact is available yet. This run establishes the initial measurement.",
176+ );
177+ } else if (!comparable) {
178+ notices.push(
179+ "> ⚠️ The thread fixture changed, so impact percentages are not directly comparable to the `main` baseline.",
180+ );
181+ } else if (!input.baselineRun.matchesBase) {
182+ notices.push(
183+ "> ℹ️ The exact PR base did not have a successful artifact. Baseline uses the latest successful `main` measurement shown below.",
184+ );
185+ }
186+ if (ceilingChanges.length > 0) {
187+ notices.push(
188+ `> ⚠️ **This PR changes transfer ceilings:**\n>\n${ceilingChanges.map((line) => `> ${line}`).join("\n")}`,
189+ );
190+ }
191+
192+ return [
193+ COMMENT_MARKER,
194+ resultShaMarker(input.currentRun.sha),
195+ "## Thread transfer impact",
196+ "",
197+ failed
198+ ? "❌ One or more thread transfer ceilings were exceeded."
199+ : "✅ Thread transfer remains within every enforced ceiling.",
200+ ...(notices.length > 0 ? ["", ...notices] : []),
201+ "",
202+ "| Provider | Metric | Main baseline | This PR | Impact | PR ceiling | |",
203+ "| --- | --- | ---: | ---: | ---: | ---: | --- |",
204+ ...rows,
205+ "",
206+ `Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`,
207+ "",
208+ "<details>",
209+ "<summary>Scenario and decoded snapshot size</summary>",
210+ "",
211+ `${current.scenario.historyTurns} historical turns, ${current.scenario.historyCommandToolsPerTurn} command tools per turn, ${formatBytes(current.scenario.historyMcpResultBytes)} retained MCP result per historical turn, and a ${formatBytes(current.scenario.measuredMcpResultBytes)} retained result in the measured turn.`,
212+ "",
213+ ...PROVIDERS.map(
214+ (provider) =>
215+ `- ${provider === "codex" ? "Codex" : "Claude"} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
216+ ),
217+ "",
218+ "</details>",
219+ "",
220+ "_Updated in place by a trusted workflow. PR artifacts are strictly validated and never executed._",
221+ ].join("\n");
222+}
223+
224+async function artifactsForRun(github, owner, repo, runId) {
225+ return github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
226+ owner,
227+ repo,
228+ run_id: runId,
229+ per_page: 100,
230+ });
231+}
232+
233+function findResultArtifact(artifacts) {
234+ return artifacts.find((artifact) => artifact.name === ARTIFACT_NAME && !artifact.expired);
235+}
236+
237+async function resolve({ github, context, core }) {
238+ const source = context.payload.workflow_run;
239+ const { owner, repo } = context.repo;
240+ if (source.event !== "pull_request") {
241+ core.setOutput("publish", "false");
242+ return;
243+ }
244+
245+ let pullNumber = source.pull_requests?.[0]?.number;
246+ if (!pullNumber) {
247+ const associated = await github.paginate(
248+ github.rest.repos.listPullRequestsAssociatedWithCommit,
249+ { owner, repo, commit_sha: source.head_sha, per_page: 100 },
250+ );
251+ const matchingPulls = associated.filter(
252+ (pull) =>
253+ pull.state === "open" &&
254+ pull.head.sha === source.head_sha &&
255+ pull.head.ref === source.head_branch,
256+ );
257+ if (matchingPulls.length !== 1) {
258+ core.info(
259+ `Expected one open pull request for ${source.head_repository?.full_name ?? "unknown repository"}:${source.head_branch ?? "unknown branch"} at ${source.head_sha}; found ${matchingPulls.length}.`,
260+ );
261+ core.setOutput("publish", "false");
262+ return;
263+ }
264+ pullNumber = matchingPulls[0].number;
265+ }
266+ if (!pullNumber) {
267+ core.info("No open pull request is associated with the completed CI run.");
268+ core.setOutput("publish", "false");
269+ return;
270+ }
271+
272+ const { data: pull } = await github.rest.pulls.get({ owner, repo, pull_number: pullNumber });
273+ if (pull.head.sha !== source.head_sha) {
274+ core.info(`Skipping stale CI result ${source.head_sha}; PR head is ${pull.head.sha}.`);
275+ core.setOutput("publish", "false");
276+ return;
277+ }
278+
279+ const sourceArtifacts = await artifactsForRun(github, owner, repo, source.id);
280+ const sourceArtifact = findResultArtifact(sourceArtifacts);
281+ const workflowRuns = await github.paginate(github.rest.actions.listWorkflowRuns, {
282+ owner,
283+ repo,
284+ workflow_id: source.workflow_id,
285+ branch: pull.base.ref,
286+ event: "push",
287+ status: "success",
288+ per_page: 100,
289+ });
290+ const orderedRuns = [
291+ ...workflowRuns.filter((run) => run.head_sha === pull.base.sha),
292+ ...workflowRuns.filter((run) => run.head_sha !== pull.base.sha),
293+ ].slice(0, 20);
294+
295+ let baselineRun;
296+ for (const run of orderedRuns) {
297+ const artifacts = await artifactsForRun(github, owner, repo, run.id);
298+ if (findResultArtifact(artifacts)) {
299+ baselineRun = run;
300+ break;
301+ }
302+ }
303+
304+ core.setOutput("publish", "true");
305+ core.setOutput("pull_number", String(pullNumber));
306+ core.setOutput("pr_artifact", sourceArtifact ? "true" : "false");
307+ core.setOutput("pr_run_id", String(source.id));
308+ core.setOutput("pr_sha", source.head_sha);
309+ core.setOutput("pr_conclusion", source.conclusion ?? "unknown");
310+ core.setOutput("baseline_artifact", baselineRun ? "true" : "false");
311+ core.setOutput("baseline_run_id", baselineRun ? String(baselineRun.id) : "");
312+ core.setOutput("baseline_sha", baselineRun?.head_sha ?? "");
313+ core.setOutput(
314+ "baseline_matches_base",
315+ baselineRun?.head_sha === pull.base.sha ? "true" : "false",
316+ );
317+}
318+
319+async function upsertComment(github, context, pullNumber, body, options = {}) {
320+ const { owner, repo } = context.repo;
321+ const comments = await github.paginate(github.rest.issues.listComments, {
322+ owner,
323+ repo,
324+ issue_number: pullNumber,
325+ per_page: 100,
326+ });
327+ const existing = comments.find(
328+ (comment) =>
329+ comment.user?.login === "github-actions[bot]" && comment.body?.includes(COMMENT_MARKER),
330+ );
331+ if (
332+ options.preserveResultSha &&
333+ existing?.body?.includes(resultShaMarker(options.preserveResultSha))
334+ ) {
335+ return;
336+ }
337+ if (existing) {
338+ await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
339+ } else {
340+ await github.rest.issues.createComment({ owner, repo, issue_number: pullNumber, body });
341+ }
342+}
343+
344+async function upsertCommentForCurrentHead(
345+ github,
346+ context,
347+ core,
348+ pullNumber,
349+ expectedSha,
350+ body,
351+ options,
352+) {
353+ const { owner, repo } = context.repo;
354+ const { data: pull } = await github.rest.pulls.get({
355+ owner,
356+ repo,
357+ pull_number: pullNumber,
358+ });
359+ if (pull.head.sha !== expectedSha) {
360+ core.info(`Skipping stale CI result ${expectedSha}; PR head is ${pull.head.sha}.`);
361+ return false;
362+ }
363+
364+ await upsertComment(github, context, pullNumber, body, options);
365+ return true;
366+}
367+
368+async function publish({ github, context, core }) {
369+ const pullNumber = Number(process.env.PR_NUMBER);
370+ if (!Number.isSafeInteger(pullNumber) || pullNumber <= 0) {
371+ throw new Error("PR_NUMBER is invalid");
372+ }
373+
374+ const current = readResult(process.env.PR_RESULT_DIR);
375+ const currentRun = {
376+ sha: process.env.PR_SHA,
377+ conclusion: process.env.PR_CONCLUSION,
378+ url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.PR_RUN_ID}`,
379+ };
380+ if (!current) {
381+ await upsertCommentForCurrentHead(
382+ github,
383+ context,
384+ core,
385+ pullNumber,
386+ currentRun.sha,
387+ [
388+ COMMENT_MARKER,
389+ "## Thread transfer impact",
390+ "",
391+ `⚠️ The latest [CI run](${currentRun.url}) did not produce a thread transfer result for \`${currentRun.sha.slice(0, 7)}\`.`,
392+ "",
393+ "_This comment will update automatically after the next completed run._",
394+ ].join("\n"),
395+ { preserveResultSha: currentRun.sha },
396+ );
397+ return;
398+ }
399+
400+ const baseline = readResult(process.env.BASELINE_RESULT_DIR);
401+ const baselineRun = baseline
402+ ? {
403+ sha: process.env.BASELINE_SHA,
404+ matchesBase: process.env.BASELINE_MATCHES_BASE === "true",
405+ url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`,
406+ }
407+ : undefined;
408+ const body = renderComment({ current, baseline, currentRun, baselineRun });
409+ const published = await upsertCommentForCurrentHead(
410+ github,
411+ context,
412+ core,
413+ pullNumber,
414+ currentRun.sha,
415+ body,
416+ );
417+ if (published) {
418+ core.info(`Updated thread transfer report on PR #${pullNumber}.`);
419+ }
420+}
421+
422+module.exports = {
423+ publish,
424+ readResult,
425+ renderComment,
426+ resolve,
427+ upsertCommentForCurrentHead,
428+ validateResult,
429+};
.github/scripts/thread-transfer-report.test.cjs+292−0
@@ -0,0 +1,292 @@
1+const assert = require("node:assert/strict");
2+const test = require("node:test");
3+
4+const {
5+ renderComment,
6+ resolve,
7+ upsertCommentForCurrentHead,
8+ validateResult,
9+} = require("./thread-transfer-report.cjs");
10+
11+function result(overrides = {}) {
12+ const observed = {
13+ totalWireBytes: 2_200_000,
14+ threadSnapshotWireBytes: 1_950_000,
15+ threadSnapshotDecodedBytes: 9_100_000,
16+ measuredTurnWebSocketWireBytes: 250_000,
17+ measuredTurnWebSocketDecodedBytes: 1_150_000,
18+ measuredTurnWebSocketMessages: 15,
19+ };
20+ const ceiling = {
21+ totalWireBytes: 2_900_000,
22+ threadSnapshotWireBytes: 2_600_000,
23+ measuredTurnWebSocketWireBytes: 320_000,
24+ measuredTurnWebSocketDecodedBytes: 1_550_000,
25+ measuredTurnWebSocketMessages: 20,
26+ };
27+ return {
28+ schemaVersion: 1,
29+ scenario: {
30+ id: "thread-transfer-v1",
31+ historyTurns: 10,
32+ historyCommandToolsPerTurn: 5,
33+ historyMcpResultBytes: 900_000,
34+ measuredCommandTools: 20,
35+ measuredMcpResultBytes: 1_100_000,
36+ },
37+ providers: {
38+ codex: { observed: { ...observed, ...overrides }, ceiling },
39+ claudeAgent: { observed, ceiling },
40+ },
41+ };
42+}
43+
44+test("validates the fixed artifact schema", () => {
45+ assert.equal(validateResult(result()).schemaVersion, 1);
46+ assert.throws(
47+ () => validateResult({ ...result(), injectedMarkdown: "@everyone" }),
48+ /unexpected fields/,
49+ );
50+ assert.throws(
51+ () => validateResult(result({ totalWireBytes: "lots" })),
52+ /non-negative safe integer/,
53+ );
54+});
55+
56+test("renders baseline, impact, ceiling, and ceiling changes", () => {
57+ const baseline = result();
58+ const current = result({ measuredTurnWebSocketWireBytes: 260_000 });
59+ current.providers.codex.ceiling = {
60+ ...current.providers.codex.ceiling,
61+ measuredTurnWebSocketWireBytes: 330_000,
62+ };
63+ const comment = renderComment({
64+ current,
65+ baseline,
66+ currentRun: {
67+ sha: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
68+ conclusion: "success",
69+ url: "https://github.com/pingdotgg/t3code/actions/runs/2",
70+ },
71+ baselineRun: {
72+ sha: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
73+ matchesBase: true,
74+ url: "https://github.com/pingdotgg/t3code/actions/runs/1",
75+ },
76+ });
77+
78+ assert.match(comment, /Main baseline \| This PR \| Impact \| PR ceiling/);
79+ assert.match(comment, /\+9\.8 KiB \(\+4\.0%\)/);
80+ assert.match(comment, /This PR changes transfer ceilings/);
81+ assert.match(comment, /312\.5 KiB → 322\.3 KiB/);
82+ assert.match(comment, /<!-- t3-thread-transfer-report -->/);
83+ assert.match(
84+ comment,
85+ /<!-- t3-thread-transfer-result-sha:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb -->/,
86+ );
87+});
88+
89+test("resolves a fallback PR with a redacted head repo and exact main baseline", async () => {
90+ const outputs = {};
91+ const listWorkflowRunArtifacts = () => {};
92+ const listWorkflowRuns = () => {};
93+ const listPullRequestsAssociatedWithCommit = () => {};
94+ const github = {
95+ paginate: async (method, input) => {
96+ if (method === listPullRequestsAssociatedWithCommit) {
97+ return [
98+ {
99+ number: 5350,
100+ state: "open",
101+ head: { sha: "head-sha", ref: "feature-branch", repo: null },
102+ },
103+ ];
104+ }
105+ if (method === listWorkflowRunArtifacts) {
106+ return [
107+ {
108+ name: "thread-transfer-results",
109+ expired: false,
110+ runId: input.run_id,
111+ },
112+ ];
113+ }
114+ if (method === listWorkflowRuns) {
115+ return [{ id: 1, head_sha: "base-sha" }];
116+ }
117+ throw new Error("unexpected pagination call");
118+ },
119+ rest: {
120+ actions: { listWorkflowRunArtifacts, listWorkflowRuns },
121+ pulls: {
122+ get: async () => ({
123+ data: {
124+ head: { sha: "head-sha" },
125+ base: { sha: "base-sha", ref: "main" },
126+ },
127+ }),
128+ },
129+ repos: { listPullRequestsAssociatedWithCommit },
130+ },
131+ };
132+ await resolve({
133+ github,
134+ context: {
135+ repo: { owner: "pingdotgg", repo: "t3code" },
136+ payload: {
137+ workflow_run: {
138+ id: 2,
139+ event: "pull_request",
140+ workflow_id: 3,
141+ head_sha: "head-sha",
142+ head_branch: "feature-branch",
143+ head_repository: { full_name: "pingdotgg/t3code" },
144+ conclusion: "success",
145+ pull_requests: [],
146+ },
147+ },
148+ },
149+ core: {
150+ info: () => {},
151+ setOutput: (key, value) => {
152+ outputs[key] = value;
153+ },
154+ },
155+ });
156+
157+ assert.equal(outputs.publish, "true");
158+ assert.equal(outputs.pull_number, "5350");
159+ assert.equal(outputs.pr_artifact, "true");
160+ assert.equal(outputs.baseline_run_id, "1");
161+ assert.equal(outputs.baseline_matches_base, "true");
162+});
163+
164+test("does not guess when a fallback commit belongs to multiple PRs", async () => {
165+ const outputs = {};
166+ const listPullRequestsAssociatedWithCommit = () => {};
167+ let fetchedPull = false;
168+ await resolve({
169+ github: {
170+ paginate: async (method) => {
171+ assert.equal(method, listPullRequestsAssociatedWithCommit);
172+ return [5350, 5351].map((number) => ({
173+ number,
174+ state: "open",
175+ head: {
176+ sha: "head-sha",
177+ ref: "feature-branch",
178+ repo: { full_name: "pingdotgg/t3code" },
179+ },
180+ }));
181+ },
182+ rest: {
183+ actions: {},
184+ pulls: {
185+ get: async () => {
186+ fetchedPull = true;
187+ },
188+ },
189+ repos: { listPullRequestsAssociatedWithCommit },
190+ },
191+ },
192+ context: {
193+ repo: { owner: "pingdotgg", repo: "t3code" },
194+ payload: {
195+ workflow_run: {
196+ id: 2,
197+ event: "pull_request",
198+ workflow_id: 3,
199+ head_sha: "head-sha",
200+ head_branch: "feature-branch",
201+ head_repository: { full_name: "pingdotgg/t3code" },
202+ conclusion: "success",
203+ pull_requests: [],
204+ },
205+ },
206+ },
207+ core: {
208+ info: () => {},
209+ setOutput: (key, value) => {
210+ outputs[key] = value;
211+ },
212+ },
213+ });
214+
215+ assert.equal(outputs.publish, "false");
216+ assert.equal(fetchedPull, false);
217+});
218+
219+test("does not publish a stale result after the PR head advances", async () => {
220+ let listedComments = false;
221+ const info = [];
222+ const published = await upsertCommentForCurrentHead(
223+ {
224+ paginate: async () => {
225+ listedComments = true;
226+ return [];
227+ },
228+ rest: {
229+ issues: {
230+ listComments: () => {},
231+ createComment: () => {
232+ throw new Error("must not create a stale comment");
233+ },
234+ updateComment: () => {
235+ throw new Error("must not update a stale comment");
236+ },
237+ },
238+ pulls: {
239+ get: async () => ({ data: { head: { sha: "new-head-sha" } } }),
240+ },
241+ },
242+ },
243+ { repo: { owner: "pingdotgg", repo: "t3code" } },
244+ { info: (message) => info.push(message) },
245+ 5350,
246+ "old-head-sha",
247+ "stale body",
248+ );
249+
250+ assert.equal(published, false);
251+ assert.equal(listedComments, false);
252+ assert.deepEqual(info, ["Skipping stale CI result old-head-sha; PR head is new-head-sha."]);
253+});
254+
255+test("preserves a successful result when a same-SHA rerun has no artifact", async () => {
256+ let updatedComment = false;
257+ const sha = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
258+ const published = await upsertCommentForCurrentHead(
259+ {
260+ paginate: async () => [
261+ {
262+ id: 1,
263+ user: { login: "github-actions[bot]" },
264+ body: `<!-- t3-thread-transfer-report -->\n<!-- t3-thread-transfer-result-sha:${sha} -->`,
265+ },
266+ ],
267+ rest: {
268+ issues: {
269+ listComments: () => {},
270+ createComment: () => {
271+ updatedComment = true;
272+ },
273+ updateComment: () => {
274+ updatedComment = true;
275+ },
276+ },
277+ pulls: {
278+ get: async () => ({ data: { head: { sha } } }),
279+ },
280+ },
281+ },
282+ { repo: { owner: "pingdotgg", repo: "t3code" } },
283+ { info: () => {} },
284+ 5350,
285+ sha,
286+ "missing artifact warning",
287+ { preserveResultSha: sha },
288+ );
289+
290+ assert.equal(published, true);
291+ assert.equal(updatedComment, false);
292+});
.github/workflows/ci.yml+21−0
@@ -84,8 +84,29 @@ jobs: · 83 unmodified lines
8484 run: vp run --filter @t3tools/desktop ensure:electron
8585
8686 - name: Test
87+ env:
88+ T3CODE_TRANSFER_BUDGET_REPORT_PATH: ${{ runner.temp }}/t3code-transfer-budget.md
89+ T3CODE_TRANSFER_BUDGET_RESULT_PATH: ${{ runner.temp }}/thread-transfer-result.json
8790 run: vp run test
8891
92+ - name: Publish transfer budget report
93+ if: always()
94+ run: |
95+ if test -f "${{ runner.temp }}/t3code-transfer-budget.md"; then
96+ tee -a "$GITHUB_STEP_SUMMARY" < "${{ runner.temp }}/t3code-transfer-budget.md"
97+ else
98+ echo "Transfer budget report was not produced." >> "$GITHUB_STEP_SUMMARY"
99+ fi
100+
101+ - name: Upload thread transfer result
102+ if: always()
103+ uses: actions/upload-artifact@v7
104+ with:
105+ name: thread-transfer-results
106+ path: ${{ runner.temp }}/thread-transfer-result.json
107+ if-no-files-found: ignore
108+ retention-days: 30
109+
89110 - name: Test resource monitor
90111 run: cargo test --locked --manifest-path native/resource-monitor/Cargo.toml
91112
.github/workflows/thread-transfer-report.yml+75−0
@@ -0,0 +1,75 @@
1+name: Thread Transfer Report
2+
3+on:
4+ workflow_run:
5+ workflows: [CI]
6+ types: [completed]
7+
8+permissions:
9+ actions: read
10+ contents: read
11+ pull-requests: write
12+
13+jobs:
14+ publish:
15+ name: Publish PR comment
16+ if: github.event.workflow_run.event == 'pull_request'
17+ runs-on: ubuntu-24.04
18+ concurrency:
19+ group: thread-transfer-report-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.id }}
20+ cancel-in-progress: true
21+ steps:
22+ # workflow_run has a write-capable token even for fork PRs. Only load the
23+ # publisher from the trusted default branch and never execute PR code.
24+ - name: Checkout trusted publisher
25+ uses: actions/checkout@v6
26+ with:
27+ ref: ${{ github.event.repository.default_branch }}
28+ sparse-checkout: .github/scripts
29+
30+ - name: Test trusted publisher
31+ run: node --test .github/scripts/thread-transfer-report.test.cjs
32+
33+ - id: resolve
34+ name: Resolve PR and baseline artifacts
35+ uses: actions/github-script@v8
36+ with:
37+ script: |
38+ const reporter = require("./.github/scripts/thread-transfer-report.cjs");
39+ await reporter.resolve({ github, context, core });
40+
41+ - name: Download PR result
42+ if: steps.resolve.outputs.publish == 'true' && steps.resolve.outputs.pr_artifact == 'true'
43+ uses: actions/download-artifact@v8
44+ with:
45+ name: thread-transfer-results
46+ path: ${{ runner.temp }}/thread-transfer/pr
47+ github-token: ${{ secrets.GITHUB_TOKEN }}
48+ run-id: ${{ steps.resolve.outputs.pr_run_id }}
49+
50+ - name: Download main baseline
51+ if: steps.resolve.outputs.publish == 'true' && steps.resolve.outputs.baseline_artifact == 'true'
52+ uses: actions/download-artifact@v8
53+ with:
54+ name: thread-transfer-results
55+ path: ${{ runner.temp }}/thread-transfer/main
56+ github-token: ${{ secrets.GITHUB_TOKEN }}
57+ run-id: ${{ steps.resolve.outputs.baseline_run_id }}
58+
59+ - name: Update thread transfer comment
60+ if: steps.resolve.outputs.publish == 'true'
61+ uses: actions/github-script@v8
62+ env:
63+ PR_NUMBER: ${{ steps.resolve.outputs.pull_number }}
64+ PR_SHA: ${{ steps.resolve.outputs.pr_sha }}
65+ PR_CONCLUSION: ${{ steps.resolve.outputs.pr_conclusion }}
66+ PR_RUN_ID: ${{ steps.resolve.outputs.pr_run_id }}
67+ PR_RESULT_DIR: ${{ runner.temp }}/thread-transfer/pr
68+ BASELINE_SHA: ${{ steps.resolve.outputs.baseline_sha }}
69+ BASELINE_MATCHES_BASE: ${{ steps.resolve.outputs.baseline_matches_base }}
70+ BASELINE_RUN_ID: ${{ steps.resolve.outputs.baseline_run_id }}
71+ BASELINE_RESULT_DIR: ${{ runner.temp }}/thread-transfer/main
72+ with:
73+ script: |
74+ const reporter = require("./.github/scripts/thread-transfer-report.cjs");
75+ await reporter.publish({ github, context, core });