History

handle source sync failures without 500s

Rhys Sullivan committed 7 months ago

Commit 95dcccd1 parent 87ff1dd2 files · +155 −3Browse files

Unified · Split

2 files
  1. packages/platform/sdk/src/sources/operations.test.ts +148 −0
  2. packages/platform/sdk/src/sources/operations.ts +7 −3
packages/platform/sdk/src/sources/operations.test.ts+148−0
@@ -0,0 +1,148 @@
1+import {
2+ mkdtempSync,
3+ rmSync,
4+} from "node:fs";
5+import {
6+ tmpdir,
7+} from "node:os";
8+import {
9+ join,
10+} from "node:path";
11+import {
12+ describe,
13+ expect,
14+ it,
15+} from "@effect/vitest";
16+import * as Effect from "effect/Effect";
17+import * as Schema from "effect/Schema";
18+
19+import {
20+ createLocalExecutorEffect,
21+} from "../../../sdk-file/src/index";
22+import {
23+ defineExecutorSourcePlugin,
24+} from "../plugins";
25+
26+const BrokenSourceInputSchema = Schema.Struct({
27+ name: Schema.String,
28+});
29+
30+type BrokenSourceInput = typeof BrokenSourceInputSchema.Type;
31+type BrokenStoredSource = {
32+ readonly name: string;
33+};
34+
35+const makeBrokenSourcePlugin = () => {
36+ const storage = new Map<string, BrokenStoredSource>();
37+
38+ return defineExecutorSourcePlugin<
39+ "broken",
40+ BrokenSourceInput,
41+ BrokenSourceInput,
42+ BrokenSourceInput,
43+ BrokenStoredSource,
44+ {
45+ sourceId: string;
46+ config: BrokenSourceInput;
47+ },
48+ {
49+ createSource: (input: BrokenSourceInput) => Effect.Effect<any, Error, never>;
50+ getSource: (sourceId: string) => Effect.Effect<any, Error, never>;
51+ }
52+ >({
53+ key: "broken",
54+ source: {
55+ kind: "broken",
56+ displayName: "Broken",
57+ add: {
58+ inputSchema: BrokenSourceInputSchema,
59+ toConnectInput: (input) => input,
60+ },
61+ storage: {
62+ get: ({ sourceId }) => Effect.succeed(storage.get(sourceId) ?? null),
63+ put: ({ sourceId, value }) =>
64+ Effect.sync(() => {
65+ storage.set(sourceId, value);
66+ }),
67+ remove: ({ sourceId }) =>
68+ Effect.sync(() => {
69+ storage.delete(sourceId);
70+ }),
71+ },
72+ source: {
73+ create: (input) => ({
74+ source: {
75+ name: input.name,
76+ kind: "broken",
77+ status: "connected",
78+ enabled: true,
79+ namespace: "broken",
80+ },
81+ stored: {
82+ name: input.name,
83+ },
84+ }),
85+ update: ({ source, config }) => ({
86+ source: {
87+ ...source,
88+ name: config.name,
89+ },
90+ stored: {
91+ name: config.name,
92+ },
93+ }),
94+ toConfig: ({ stored }) => ({
95+ name: stored.name,
96+ }),
97+ },
98+ catalog: {
99+ kind: "imported",
100+ sync: () => Effect.fail(new Error("sync boom")),
101+ invoke: () => Effect.fail(new Error("invoke should not run in this test")),
102+ },
103+ },
104+ extendExecutor: ({ source }) => ({
105+ createSource: (input) => source.createSource(input),
106+ getSource: (sourceId) => source.getSource(sourceId),
107+ }),
108+ });
109+};
110+
111+describe("source operations", () => {
112+ it.scoped("returns the errored source when source sync fails during create", () =>
113+ Effect.gen(function* () {
114+ const workspaceRoot = mkdtempSync(join(tmpdir(), "executor-source-ops-"));
115+
116+ const executor = yield* Effect.acquireRelease(
117+ createLocalExecutorEffect({
118+ localDataDir: ":memory:",
119+ workspaceRoot,
120+ plugins: [makeBrokenSourcePlugin()] as const,
121+ }),
122+ (executor) =>
123+ Effect.promise(() => executor.close()).pipe(
124+ Effect.orDie,
125+ Effect.zipRight(
126+ Effect.sync(() => {
127+ rmSync(workspaceRoot, {
128+ recursive: true,
129+ force: true,
130+ });
131+ }),
132+ ),
133+ ),
134+ );
135+
136+ const created = yield* executor.broken.createSource({
137+ name: "Broken Source",
138+ });
139+
140+ expect(created.status).toBe("error");
141+
142+ const persisted = yield* executor.broken.getSource(created.id);
143+
144+ expect(persisted.status).toBe("error");
145+ expect(persisted.name).toBe("Broken Source");
146+ }),
147+ );
148+});
packages/platform/sdk/src/sources/operations.ts+7−3
@@ -79,11 +79,15 @@ const syncArtifactsForSource = (input: { · 78 unmodified lines
7979 actorScopeId: input.actorScopeId,
8080 }),
8181 );
82+
83+ return erroredSource;
8284 }
8385
84− return yield* input.operation.unknownStorage(
85− error,
86− "Failed syncing source tools",
86+ return yield* Effect.fail(
87+ input.operation.unknownStorage(
88+ error,
89+ "Failed syncing source tools",
90+ ),
8791 );
8892 }),
8993 });