History

Harden file secrets migration

Rhys Sullivan committed 4 months ago

Commit 5130fa41 parent af1905c3 files · +128 −32Browse files

Unified · Split

3 files
  1. packages/plugins/file-secrets/src/data-dir.test.ts +2 −5
  2. packages/plugins/file-secrets/src/index.test.ts +73 −9
  3. packages/plugins/file-secrets/src/index.ts +53 −18
packages/plugins/file-secrets/src/data-dir.test.ts+2−5
@@ -1,7 +1,7 @@
11 import { afterEach, beforeEach, describe, expect, it, vi } from "@effect/vitest";
22 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
33 import { tmpdir } from "node:os";
4−import { isAbsolute, join, relative } from "node:path";
4+import { join } from "node:path";
55 import { Effect } from "effect";
66
77 import {
@@ -81,10 +81,7 @@ describe("file secrets data directory", () => { · 73 unmodified lines
8181
8282 expect(connection.provider).toBe(ProviderKey.make("file"));
8383 const authPath = first.executor.fileSecrets.filePath;
84− const pathWithinDataDir = relative(dataDir, authPath);
85− if (pathWithinDataDir.startsWith("..") || isAbsolute(pathWithinDataDir)) {
86− expect(authPath).toBe(join(firstSandboxDataHome, "executor", "auth.json"));
87− }
84+ expect(authPath).toBe(join(dataDir, "auth.json"));
8885 expect(existsSync(authPath)).toBe(true);
8986 expect(readFileSync(authPath, "utf8")).toContain(
9087 '"connection:org:durable-secrets:main:token": "secret-token"',
packages/plugins/file-secrets/src/index.test.ts+73−9
@@ -4,13 +4,14 @@ import { · 3 unmodified lines
44 mkdirSync,
55 mkdtempSync,
66 readFileSync,
7+ readdirSync,
78 rmSync,
89 statSync,
910 writeFileSync,
1011 } from "node:fs";
1112 import { tmpdir } from "node:os";
1213 import { dirname, join } from "node:path";
13−import { Effect } from "effect";
14+import { Effect, Predicate, Result } from "effect";
1415
1516 import { ProviderKey } from "@executor-js/sdk";
1617 import { makeTestWorkspaceHarness } from "@executor-js/sdk/testing";
@@ -130,15 +131,78 @@ describe("file secrets auth location", () => { · 113 unmodified lines
130131 );
131132
132133 it.effect("leaves the new store empty when the legacy file is corrupt", () =>
133− Effect.gen(function* () {
134− vi.stubEnv("EXECUTOR_DATA_DIR", dataDir);
135− writeAuthFile(legacyFilePath, "not-json");
134+ Effect.scoped(
135+ Effect.gen(function* () {
136+ vi.stubEnv("EXECUTOR_DATA_DIR", dataDir);
137+ writeAuthFile(legacyFilePath, "not-json");
138+ const workspace = yield* makeTestWorkspaceHarness({
139+ plugins: [fileSecretsPlugin()] as const,
140+ });
141+
142+ const initial = yield* workspace.executor.providers.items(FILE_PROVIDER);
143+ expect(initial).toEqual([]);
144+ expect(existsSync(join(dataDir, "auth.json"))).toBe(false);
145+ expect(readFileSync(legacyFilePath, "utf8")).toBe("not-json");
146+
147+ writeAuthFile(legacyFilePath, '{"repaired-token":"repaired-secret"}');
148+ const afterRepair = yield* workspace.executor.providers.items(FILE_PROVIDER);
149+ expect(afterRepair).toEqual([]);
150+ expect(existsSync(join(dataDir, "auth.json"))).toBe(false);
151+ }),
152+ ),
153+ );
136154
137− const inspected = yield* inspectPlugin(fileSecretsPlugin());
155+ it.effect("retries migration after a legacy read I/O failure", () =>
156+ Effect.scoped(
157+ Effect.gen(function* () {
158+ vi.stubEnv("EXECUTOR_DATA_DIR", dataDir);
159+ mkdirSync(legacyFilePath, { recursive: true });
160+ const workspace = yield* makeTestWorkspaceHarness({
161+ plugins: [fileSecretsPlugin()] as const,
162+ });
163+
164+ const failed = yield* Effect.result(workspace.executor.providers.items(FILE_PROVIDER));
165+ expect(Result.isFailure(failed)).toBe(true);
166+ if (!Result.isFailure(failed)) return;
167+ expect(Predicate.isTagged("StorageError")(failed.failure)).toBe(true);
168+
169+ rmSync(legacyFilePath, { recursive: true, force: true });
170+ writeAuthFile(legacyFilePath, '{"recovered-token":"recovered-secret"}');
171+
172+ const recovered = yield* workspace.executor.providers.items(FILE_PROVIDER);
173+ expect(recovered.map((item) => String(item.id))).toEqual(["recovered-token"]);
174+ expect(readFileSync(join(dataDir, "auth.json"), "utf8")).toContain(
175+ '"recovered-token": "recovered-secret"',
176+ );
177+ }),
178+ ),
179+ );
138180
139− expect(inspected.itemIds).toEqual([]);
140− expect(existsSync(join(dataDir, "auth.json"))).toBe(false);
141− expect(readFileSync(legacyFilePath, "utf8")).toBe("not-json");
142− }),
181+ it.effect("shares one migration across concurrent first provider operations", () =>
182+ Effect.scoped(
183+ Effect.gen(function* () {
184+ vi.stubEnv("EXECUTOR_DATA_DIR", dataDir);
185+ const legacyContents = '{"legacy-token":"legacy-secret"}';
186+ writeAuthFile(legacyFilePath, legacyContents);
187+ const workspace = yield* makeTestWorkspaceHarness({
188+ plugins: [fileSecretsPlugin()] as const,
189+ });
190+
191+ const results = yield* Effect.all(
192+ [
193+ workspace.executor.providers.items(FILE_PROVIDER),
194+ workspace.executor.providers.items(FILE_PROVIDER),
195+ ],
196+ { concurrency: "unbounded" },
197+ );
198+
199+ expect(results.map((items) => items.map((item) => String(item.id)))).toEqual([
200+ ["legacy-token"],
201+ ["legacy-token"],
202+ ]);
203+ expect(readdirSync(dataDir)).toEqual(["auth.json"]);
204+ expect(readFileSync(legacyFilePath, "utf8")).toBe(legacyContents);
205+ }),
206+ ),
143207 );
144208 });
packages/plugins/file-secrets/src/index.ts+53−18
@@ -1,6 +1,7 @@
1−import { Effect, Schema } from "effect";
1+import { randomUUID } from "node:crypto";
22 import * as fs from "node:fs";
33 import * as path from "node:path";
4+import { Deferred, Effect, Exit, Schema } from "effect";
45
56 import {
67 definePlugin,
@@ -71,13 +72,29 @@ const resolveAuthLocation = (config: FileSecretsPluginConfig | undefined): AuthL · 64 unmodified lines
7172 const FlatAuthFile = Schema.Record(Schema.String, Schema.String);
7273 const decodeFlatAuthFile = Schema.decodeUnknownEffect(Schema.fromJsonString(FlatAuthFile));
7374
75+class AuthFileDecodeError extends Schema.TaggedErrorClass<AuthFileDecodeError>()(
76+ "AuthFileDecodeError",
77+ {
78+ filePath: Schema.String,
79+ cause: Schema.Defect,
80+ },
81+ {
82+ description:
83+ "The auth file was readable but its contents were not a valid flat credential map. The caller must surface the failure or explicitly treat malformed legacy data as non-migratable.",
84+ },
85+) {
86+ override get message(): string {
87+ return `Failed to parse auth file: ${this.filePath}`;
88+ }
89+}
90+
7491 // ---------------------------------------------------------------------------
7592 // File I/O with restricted permissions
7693 //
77−// These helpers keep real I/O and decode failures in the Effect error
78−// channel as `StorageError`. Missing files are still treated as an empty
79−// auth file, but malformed JSON, schema decode failures, and permission
80−// errors no longer collapse into "empty file".
94+// These helpers keep real I/O and decode failures distinct internally, then
95+// project both to `StorageError` at the provider boundary. Missing files are
96+// still treated as an empty auth file; malformed content and permission errors
97+// do not collapse into "empty file" during ordinary provider reads.
8198 // ---------------------------------------------------------------------------
8299
83100 const isFileNotFoundCause = (cause: unknown): cause is NodeJS.ErrnoException =>
@@ -88,7 +105,9 @@ const toStorageError = · 4 unmodified lines
88105 (cause: unknown): StorageError =>
89106 new StorageError({ message, cause });
90107
91−const readAll = (filePath: string): Effect.Effect<Record<string, string>, StorageError> => {
108+const readAuthFile = (
109+ filePath: string,
110+): Effect.Effect<Record<string, string>, StorageError | AuthFileDecodeError> => {
92111 if (!fs.existsSync(filePath)) return Effect.succeed({});
93112 return Effect.try({
94113 try: () => fs.readFileSync(filePath, "utf-8"),
@@ -102,18 +121,25 @@ const readAll = (filePath: string): Effect.Effect<Record<string, string>, Storag · 7 unmodified lines
102121 raw === ""
103122 ? Effect.succeed<Record<string, string>>({})
104123 : decodeFlatAuthFile(raw).pipe(
105− Effect.mapError(toStorageError("Failed to parse auth file")),
124+ Effect.mapError((cause) => new AuthFileDecodeError({ filePath, cause })),
106125 ),
107126 ),
108127 );
109128 };
110129
130+const readAll = (filePath: string): Effect.Effect<Record<string, string>, StorageError> =>
131+ readAuthFile(filePath).pipe(
132+ Effect.catchTag("AuthFileDecodeError", (cause) =>
133+ Effect.fail(new StorageError({ message: cause.message, cause })),
134+ ),
135+ );
136+
111137 const writeAll = (
112138 filePath: string,
113139 secrets: Record<string, string>,
114140 ): Effect.Effect<void, StorageError> => {
115141 const dir = path.dirname(filePath);
116− const tmp = `${filePath}.tmp`;
142+ const tmp = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
117143 return Effect.gen(function* () {
118144 if (!fs.existsSync(dir)) {
119145 yield* Effect.try({
@@ -124,6 +150,8 @@ const writeAll = ( · 4 unmodified lines
124150 yield* Effect.try({
125151 try: () => {
126152 fs.writeFileSync(tmp, JSON.stringify(secrets, null, 2), { mode: 0o600 });
153+ // `mode` only applies when the file is created; chmod afterwards covers
154+ // the case where a temporary file with broader permissions already exists.
127155 fs.chmodSync(tmp, 0o600);
128156 },
129157 catch: toStorageError("Failed to write temporary auth file"),
@@ -143,13 +171,11 @@ const migrateLegacyAuthFile = ({ · 13 unmodified lines
143171 return Effect.void;
144172 }
145173
146− return readAll(legacyFilePath).pipe(
147− Effect.matchEffect({
148− // A legacy file that cannot be read or decoded must not block startup.
149− // The active data-dir store remains empty and all later I/O uses it only.
150− onFailure: () => Effect.void,
151− onSuccess: (secrets) => writeAll(filePath, secrets),
152− }),
174+ return readAuthFile(legacyFilePath).pipe(
175+ Effect.flatMap((secrets) => writeAll(filePath, secrets)),
176+ // Malformed legacy contents are deliberately skipped. Genuine read I/O
177+ // failures stay visible and leave migration eligible for a later retry.
178+ Effect.catchTag("AuthFileDecodeError", () => Effect.void),
153179 );
154180 };
155181
@@ -184,12 +210,21 @@ const FILE_PROVIDER_KEY = ProviderKey.make("file"); · 28 unmodified lines
184210
185211 const makeFileProvider = (location: AuthLocation): CredentialProvider => {
186212 let migrationComplete = false;
213+ let migrationInFlight: Deferred.Deferred<void, StorageError> | null = null;
187214 const ensureMigration = Effect.suspend(() => {
188215 if (migrationComplete) return Effect.void;
216+ if (migrationInFlight !== null) return Deferred.await(migrationInFlight);
217+
218+ // Installing the latch inside Effect.suspend is synchronous, so a peer
219+ // fiber can only observe either no attempt or this fully initialized one.
220+ const latch = Deferred.makeUnsafe<void, StorageError>();
221+ migrationInFlight = latch;
189222 return migrateLegacyAuthFile(location).pipe(
190− Effect.tap(() =>
191− Effect.sync(() => {
192− migrationComplete = true;
223+ Effect.onExit((exit) =>
224+ Effect.gen(function* () {
225+ if (Exit.isSuccess(exit)) migrationComplete = true;
226+ yield* Deferred.done(latch, exit);
227+ migrationInFlight = null;
193228 }),
194229 ),
195230 );

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t