History

Harden file secrets migration

Rhys Sullivan committed 4 months ago

Commit 5130fa41 parent af1905c3 files · +128 −32Browse files

Unified · Split

3 files
  1. packages/plugins/file-secrets/src/data-dir.test.ts +2 −5
  2. packages/plugins/file-secrets/src/index.test.ts +73 −9
  3. packages/plugins/file-secrets/src/index.ts +53 −18
packages/plugins/file-secrets/src/data-dir.test.ts+2−5
@@ -1,7 +1,7 @@
1import { afterEach, beforeEach, describe, expect, it, vi } from "@effect/vitest";1import { afterEach, beforeEach, describe, expect, it, vi } from "@effect/vitest";
2import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";2import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
3import { tmpdir } from "node:os";3import { tmpdir } from "node:os";
4import { isAbsolute, join, relative } from "node:path";4import { join } from "node:path";
5import { Effect } from "effect";5import { Effect } from "effect";
6 6
7import {7import {
@@ -81,10 +81,7 @@ describe("file secrets data directory", () => { · 73 unmodified lines
81 81
82 expect(connection.provider).toBe(ProviderKey.make("file"));82 expect(connection.provider).toBe(ProviderKey.make("file"));
83 const authPath = first.executor.fileSecrets.filePath;83 const authPath = first.executor.fileSecrets.filePath;
84 const pathWithinDataDir = relative(dataDir, authPath);84 expect(authPath).toBe(join(dataDir, "auth.json"));
85 if (pathWithinDataDir.startsWith("..") || isAbsolute(pathWithinDataDir)) {
86 expect(authPath).toBe(join(firstSandboxDataHome, "executor", "auth.json"));
87 }
88 expect(existsSync(authPath)).toBe(true);85 expect(existsSync(authPath)).toBe(true);
89 expect(readFileSync(authPath, "utf8")).toContain(86 expect(readFileSync(authPath, "utf8")).toContain(
90 '"connection:org:durable-secrets:main:token": "secret-token"',87 '"connection:org:durable-secrets:main:token": "secret-token"',
packages/plugins/file-secrets/src/index.test.ts+73−9
@@ -4,13 +4,14 @@ import { · 3 unmodified lines
4 mkdirSync,4 mkdirSync,
5 mkdtempSync,5 mkdtempSync,
6 readFileSync,6 readFileSync,
7 readdirSync,
7 rmSync,8 rmSync,
8 statSync,9 statSync,
9 writeFileSync,10 writeFileSync,
10} from "node:fs";11} from "node:fs";
11import { tmpdir } from "node:os";12import { tmpdir } from "node:os";
12import { dirname, join } from "node:path";13import { dirname, join } from "node:path";
13import { Effect } from "effect";14import { Effect, Predicate, Result } from "effect";
14 15
15import { ProviderKey } from "@executor-js/sdk";16import { ProviderKey } from "@executor-js/sdk";
16import { makeTestWorkspaceHarness } from "@executor-js/sdk/testing";17import { makeTestWorkspaceHarness } from "@executor-js/sdk/testing";
@@ -130,15 +131,78 @@ describe("file secrets auth location", () => { · 113 unmodified lines
130 );131 );
131 132
132 it.effect("leaves the new store empty when the legacy file is corrupt", () =>133 it.effect("leaves the new store empty when the legacy file is corrupt", () =>
133 Effect.gen(function* () {134 Effect.scoped(
134 vi.stubEnv("EXECUTOR_DATA_DIR", dataDir);135 Effect.gen(function* () {
135 writeAuthFile(legacyFilePath, "not-json");136 vi.stubEnv("EXECUTOR_DATA_DIR", dataDir);
137 writeAuthFile(legacyFilePath, "not-json");
138 const workspace = yield* makeTestWorkspaceHarness({
139 plugins: [fileSecretsPlugin()] as const,
140 });
141
142 const initial = yield* workspace.executor.providers.items(FILE_PROVIDER);
143 expect(initial).toEqual([]);
144 expect(existsSync(join(dataDir, "auth.json"))).toBe(false);
145 expect(readFileSync(legacyFilePath, "utf8")).toBe("not-json");
146
147 writeAuthFile(legacyFilePath, '{"repaired-token":"repaired-secret"}');
148 const afterRepair = yield* workspace.executor.providers.items(FILE_PROVIDER);
149 expect(afterRepair).toEqual([]);
150 expect(existsSync(join(dataDir, "auth.json"))).toBe(false);
151 }),
152 ),
153 );
136 154
137 const inspected = yield* inspectPlugin(fileSecretsPlugin());155 it.effect("retries migration after a legacy read I/O failure", () =>
156 Effect.scoped(
157 Effect.gen(function* () {
158 vi.stubEnv("EXECUTOR_DATA_DIR", dataDir);
159 mkdirSync(legacyFilePath, { recursive: true });
160 const workspace = yield* makeTestWorkspaceHarness({
161 plugins: [fileSecretsPlugin()] as const,
162 });
163
164 const failed = yield* Effect.result(workspace.executor.providers.items(FILE_PROVIDER));
165 expect(Result.isFailure(failed)).toBe(true);
166 if (!Result.isFailure(failed)) return;
167 expect(Predicate.isTagged("StorageError")(failed.failure)).toBe(true);
168
169 rmSync(legacyFilePath, { recursive: true, force: true });
170 writeAuthFile(legacyFilePath, '{"recovered-token":"recovered-secret"}');
171
172 const recovered = yield* workspace.executor.providers.items(FILE_PROVIDER);
173 expect(recovered.map((item) => String(item.id))).toEqual(["recovered-token"]);
174 expect(readFileSync(join(dataDir, "auth.json"), "utf8")).toContain(
175 '"recovered-token": "recovered-secret"',
176 );
177 }),
178 ),
179 );
138 180
139 expect(inspected.itemIds).toEqual([]);181 it.effect("shares one migration across concurrent first provider operations", () =>
140 expect(existsSync(join(dataDir, "auth.json"))).toBe(false);182 Effect.scoped(
141 expect(readFileSync(legacyFilePath, "utf8")).toBe("not-json");183 Effect.gen(function* () {
142 }),184 vi.stubEnv("EXECUTOR_DATA_DIR", dataDir);
185 const legacyContents = '{"legacy-token":"legacy-secret"}';
186 writeAuthFile(legacyFilePath, legacyContents);
187 const workspace = yield* makeTestWorkspaceHarness({
188 plugins: [fileSecretsPlugin()] as const,
189 });
190
191 const results = yield* Effect.all(
192 [
193 workspace.executor.providers.items(FILE_PROVIDER),
194 workspace.executor.providers.items(FILE_PROVIDER),
195 ],
196 { concurrency: "unbounded" },
197 );
198
199 expect(results.map((items) => items.map((item) => String(item.id)))).toEqual([
200 ["legacy-token"],
201 ["legacy-token"],
202 ]);
203 expect(readdirSync(dataDir)).toEqual(["auth.json"]);
204 expect(readFileSync(legacyFilePath, "utf8")).toBe(legacyContents);
205 }),
206 ),
143 );207 );
144});208});
packages/plugins/file-secrets/src/index.ts+53−18
@@ -1,6 +1,7 @@
1import { Effect, Schema } from "effect";1import { randomUUID } from "node:crypto";
2import * as fs from "node:fs";2import * as fs from "node:fs";
3import * as path from "node:path";3import * as path from "node:path";
4import { Deferred, Effect, Exit, Schema } from "effect";
4 5
5import {6import {
6 definePlugin,7 definePlugin,
@@ -71,13 +72,29 @@ const resolveAuthLocation = (config: FileSecretsPluginConfig | undefined): AuthL · 64 unmodified lines
71const FlatAuthFile = Schema.Record(Schema.String, Schema.String);72const FlatAuthFile = Schema.Record(Schema.String, Schema.String);
72const decodeFlatAuthFile = Schema.decodeUnknownEffect(Schema.fromJsonString(FlatAuthFile));73const decodeFlatAuthFile = Schema.decodeUnknownEffect(Schema.fromJsonString(FlatAuthFile));
73 74
75class AuthFileDecodeError extends Schema.TaggedErrorClass<AuthFileDecodeError>()(
76 "AuthFileDecodeError",
77 {
78 filePath: Schema.String,
79 cause: Schema.Defect,
80 },
81 {
82 description:
83 "The auth file was readable but its contents were not a valid flat credential map. The caller must surface the failure or explicitly treat malformed legacy data as non-migratable.",
84 },
85) {
86 override get message(): string {
87 return `Failed to parse auth file: ${this.filePath}`;
88 }
89}
90
74// ---------------------------------------------------------------------------91// ---------------------------------------------------------------------------
75// File I/O with restricted permissions92// File I/O with restricted permissions
76//93//
77// These helpers keep real I/O and decode failures in the Effect error94// These helpers keep real I/O and decode failures distinct internally, then
78// channel as `StorageError`. Missing files are still treated as an empty95// project both to `StorageError` at the provider boundary. Missing files are
79// auth file, but malformed JSON, schema decode failures, and permission96// still treated as an empty auth file; malformed content and permission errors
80// errors no longer collapse into "empty file".97// do not collapse into "empty file" during ordinary provider reads.
81// ---------------------------------------------------------------------------98// ---------------------------------------------------------------------------
82 99
83const isFileNotFoundCause = (cause: unknown): cause is NodeJS.ErrnoException =>100const isFileNotFoundCause = (cause: unknown): cause is NodeJS.ErrnoException =>
@@ -88,7 +105,9 @@ const toStorageError = · 4 unmodified lines
88 (cause: unknown): StorageError =>105 (cause: unknown): StorageError =>
89 new StorageError({ message, cause });106 new StorageError({ message, cause });
90 107
91const readAll = (filePath: string): Effect.Effect<Record<string, string>, StorageError> => {108const readAuthFile = (
109 filePath: string,
110): Effect.Effect<Record<string, string>, StorageError | AuthFileDecodeError> => {
92 if (!fs.existsSync(filePath)) return Effect.succeed({});111 if (!fs.existsSync(filePath)) return Effect.succeed({});
93 return Effect.try({112 return Effect.try({
94 try: () => fs.readFileSync(filePath, "utf-8"),113 try: () => fs.readFileSync(filePath, "utf-8"),
@@ -102,18 +121,25 @@ const readAll = (filePath: string): Effect.Effect<Record<string, string>, Storag · 7 unmodified lines
102 raw === ""121 raw === ""
103 ? Effect.succeed<Record<string, string>>({})122 ? Effect.succeed<Record<string, string>>({})
104 : decodeFlatAuthFile(raw).pipe(123 : decodeFlatAuthFile(raw).pipe(
105 Effect.mapError(toStorageError("Failed to parse auth file")),124 Effect.mapError((cause) => new AuthFileDecodeError({ filePath, cause })),
106 ),125 ),
107 ),126 ),
108 );127 );
109};128};
110 129
130const readAll = (filePath: string): Effect.Effect<Record<string, string>, StorageError> =>
131 readAuthFile(filePath).pipe(
132 Effect.catchTag("AuthFileDecodeError", (cause) =>
133 Effect.fail(new StorageError({ message: cause.message, cause })),
134 ),
135 );
136
111const writeAll = (137const writeAll = (
112 filePath: string,138 filePath: string,
113 secrets: Record<string, string>,139 secrets: Record<string, string>,
114): Effect.Effect<void, StorageError> => {140): Effect.Effect<void, StorageError> => {
115 const dir = path.dirname(filePath);141 const dir = path.dirname(filePath);
116 const tmp = `${filePath}.tmp`;142 const tmp = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
117 return Effect.gen(function* () {143 return Effect.gen(function* () {
118 if (!fs.existsSync(dir)) {144 if (!fs.existsSync(dir)) {
119 yield* Effect.try({145 yield* Effect.try({
@@ -124,6 +150,8 @@ const writeAll = ( · 4 unmodified lines
124 yield* Effect.try({150 yield* Effect.try({
125 try: () => {151 try: () => {
126 fs.writeFileSync(tmp, JSON.stringify(secrets, null, 2), { mode: 0o600 });152 fs.writeFileSync(tmp, JSON.stringify(secrets, null, 2), { mode: 0o600 });
153 // `mode` only applies when the file is created; chmod afterwards covers
154 // the case where a temporary file with broader permissions already exists.
127 fs.chmodSync(tmp, 0o600);155 fs.chmodSync(tmp, 0o600);
128 },156 },
129 catch: toStorageError("Failed to write temporary auth file"),157 catch: toStorageError("Failed to write temporary auth file"),
@@ -143,13 +171,11 @@ const migrateLegacyAuthFile = ({ · 13 unmodified lines
143 return Effect.void;171 return Effect.void;
144 }172 }
145 173
146 return readAll(legacyFilePath).pipe(174 return readAuthFile(legacyFilePath).pipe(
147 Effect.matchEffect({175 Effect.flatMap((secrets) => writeAll(filePath, secrets)),
148 // A legacy file that cannot be read or decoded must not block startup.176 // Malformed legacy contents are deliberately skipped. Genuine read I/O
149 // The active data-dir store remains empty and all later I/O uses it only.177 // failures stay visible and leave migration eligible for a later retry.
150 onFailure: () => Effect.void,178 Effect.catchTag("AuthFileDecodeError", () => Effect.void),
151 onSuccess: (secrets) => writeAll(filePath, secrets),
152 }),
153 );179 );
154};180};
155 181
@@ -184,12 +210,21 @@ const FILE_PROVIDER_KEY = ProviderKey.make("file"); · 28 unmodified lines
184 210
185const makeFileProvider = (location: AuthLocation): CredentialProvider => {211const makeFileProvider = (location: AuthLocation): CredentialProvider => {
186 let migrationComplete = false;212 let migrationComplete = false;
213 let migrationInFlight: Deferred.Deferred<void, StorageError> | null = null;
187 const ensureMigration = Effect.suspend(() => {214 const ensureMigration = Effect.suspend(() => {
188 if (migrationComplete) return Effect.void;215 if (migrationComplete) return Effect.void;
216 if (migrationInFlight !== null) return Deferred.await(migrationInFlight);
217
218 // Installing the latch inside Effect.suspend is synchronous, so a peer
219 // fiber can only observe either no attempt or this fully initialized one.
220 const latch = Deferred.makeUnsafe<void, StorageError>();
221 migrationInFlight = latch;
189 return migrateLegacyAuthFile(location).pipe(222 return migrateLegacyAuthFile(location).pipe(
190 Effect.tap(() =>223 Effect.onExit((exit) =>
191 Effect.sync(() => {224 Effect.gen(function* () {
192 migrationComplete = true;225 if (Exit.isSuccess(exit)) migrationComplete = true;
226 yield* Deferred.done(latch, exit);
227 migrationInFlight = null;
193 }),228 }),
194 ),229 ),
195 );230 );

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t