Add Danger check for changes to eval fixtures (#33852)

Also, revert unintentional changes to fixtures. Changes to test fixtures are intentional and necessary. Release Notes: - N/A

Oleksiy Syvokon committed 1 year ago

Commit cc0d8a41 parent 12ab53b2 files · +41 −188Browse files

Showing the changes to script (1 of 2 files). Show all

Unified · Split

@@ -94,3 +94,24 @@ for (const promptPath of modifiedPrompts) { · 93 unmodified lines
94	94	);
95	95	}
96	96	}
	97	+
	98	+const FIXTURE_CHANGE_ATTESTATION = "Changes to test fixtures are intentional and necessary.";
	99	+
	100	+const FIXTURES_PATHS = ["crates/assistant_tools/src/edit_agent/evals/fixtures"];
	101	+
	102	+const modifiedFixtures = danger.git.modified_files.filter((file) =>
	103	+ FIXTURES_PATHS.some((fixturePath) => file.includes(fixturePath)),
	104	+);
	105	+
	106	+if (modifiedFixtures.length > 0) {
	107	+ if (!body.includes(FIXTURE_CHANGE_ATTESTATION)) {
	108	+ const modifiedFixturesStr = modifiedFixtures.map((path) => "`" + path + "`").join(", ");
	109	+ fail(
	110	+ [
	111	+ `This PR modifies eval or test fixtures (${modifiedFixturesStr}), which are typically expected to remain unchanged.`,
	112	+ "If these changes are intentional and required, please add the following attestation to your PR description: ",
	113	+ `"${FIXTURE_CHANGE_ATTESTATION}"`,
	114	+ ].join("\n\n"),
	115	+ );
	116	+ }
	117	+}