| @@ -1,7 +1,9 @@ |
| 1 | import { describe, expect, it } from "@effect/vitest"; | 1 | import { describe, expect, it } from "@effect/vitest"; |
| 2 | import { | 2 | import { |
| | 3 | Cause, |
| 3 | Deferred, | 4 | Deferred, |
| 4 | Effect, | 5 | Effect, |
| | 6 | Exit, |
| 5 | Fiber, | 7 | Fiber, |
| 6 | Inspectable, | 8 | Inspectable, |
| 7 | Logger, | 9 | Logger, |
| @@ -610,6 +612,37 @@ const durabilityPlugin = (provider: CredentialProvider) => · 602 unmodified lines |
| 610 | }), | 612 | }), |
| 611 | }))(); | 613 | }))(); |
| 612 | | 614 | |
| | 615 | /** Wrap a test `FumaDb` so deletes on the `connection` table can be made to |
| | 616 | * fail on demand — the raw driver-level failure the compensating delete must |
| | 617 | * survive loudly. Transactions hand out wrapped handles too, so the guarded |
| | 618 | * delete inside the compensation transaction is covered. */ |
| | 619 | const failableConnectionDeletes = (db: FumaDb, shouldFail: () => boolean): FumaDb => { |
| | 620 | const wrap = (inner: FumaDb): FumaDb => |
| | 621 | new Proxy(inner, { |
| | 622 | get(target, prop) { |
| | 623 | if (prop === "withContext") { |
| | 624 | return (context: unknown) => |
| | 625 | wrap((target.withContext as (c: unknown) => FumaDb)(context)); |
| | 626 | } |
| | 627 | if (prop === "transaction") { |
| | 628 | return (run: (tx: FumaDb) => Promise<unknown>) => |
| | 629 | (target.transaction as (r: (tx: FumaDb) => Promise<unknown>) => Promise<unknown>)( |
| | 630 | (tx) => run(wrap(tx)), |
| | 631 | ); |
| | 632 | } |
| | 633 | if (prop === "deleteMany") { |
| | 634 | return (table: unknown, query: unknown) => |
| | 635 | shouldFail() && table === "connection" |
| | 636 | ? // oxlint-disable-next-line executor/no-promise-reject -- boundary: the proxy fakes a driver-level rejection from the raw FumaDb handle |
| | 637 | Promise.reject(new StorageError({ message: "delete refused", cause: undefined })) |
| | 638 | : (target.deleteMany as (t: unknown, q: unknown) => Promise<unknown>)(table, query); |
| | 639 | } |
| | 640 | return Reflect.get(target, prop); |
| | 641 | }, |
| | 642 | }); |
| | 643 | return wrap(db); |
| | 644 | }; |
| | 645 | |
| 613 | describe("connections.create credential-write compensation", () => { | 646 | describe("connections.create credential-write compensation", () => { |
| 614 | // Interruption is not an error: error-channel compensation never sees it. A | 647 | // Interruption is not an error: error-channel compensation never sees it. A |
| 615 | // create interrupted mid-write must still tear down what it already did — | 648 | // create interrupted mid-write must still tear down what it already did — |
| @@ -739,38 +772,6 @@ describe("connections.create credential-write compensation", () => { · 123 unmodified lines |
| 739 | it.effect("names the stranded connection when the compensating delete fails", () => | 772 | it.effect("names the stranded connection when the compensating delete fails", () => |
| 740 | Effect.gen(function* () { | 773 | Effect.gen(function* () { |
| 741 | let failRowDelete = false; | 774 | let failRowDelete = false; |
| 742 | const failConnectionDeletes = (db: FumaDb): FumaDb => { | | |
| 743 | const wrap = (inner: FumaDb): FumaDb => | | |
| 744 | new Proxy(inner, { | | |
| 745 | get(target, prop) { | | |
| 746 | if (prop === "withContext") { | | |
| 747 | return (context: unknown) => | | |
| 748 | wrap((target.withContext as (c: unknown) => FumaDb)(context)); | | |
| 749 | } | | |
| 750 | if (prop === "transaction") { | | |
| 751 | return (run: (tx: FumaDb) => Promise<unknown>) => | | |
| 752 | (target.transaction as (r: (tx: FumaDb) => Promise<unknown>) => Promise<unknown>)( | | |
| 753 | (tx) => run(wrap(tx)), | | |
| 754 | ); | | |
| 755 | } | | |
| 756 | if (prop === "deleteMany") { | | |
| 757 | return (table: unknown, query: unknown) => | | |
| 758 | failRowDelete && table === "connection" | | |
| 759 | ? // oxlint-disable-next-line executor/no-promise-reject -- boundary: the proxy fakes a driver-level rejection from the raw FumaDb handle | | |
| 760 | Promise.reject( | | |
| 761 | new StorageError({ message: "delete refused", cause: undefined }), | | |
| 762 | ) | | |
| 763 | : (target.deleteMany as (t: unknown, q: unknown) => Promise<unknown>)( | | |
| 764 | table, | | |
| 765 | query, | | |
| 766 | ); | | |
| 767 | } | | |
| 768 | return Reflect.get(target, prop); | | |
| 769 | }, | | |
| 770 | }); | | |
| 771 | return wrap(db); | | |
| 772 | }; | | |
| 773 | | | |
| 774 | const store = new Map<string, string>(); | 775 | const store = new Map<string, string>(); |
| 775 | const provider = trackingProvider(store, { | 776 | const provider = trackingProvider(store, { |
| 776 | set: (id, value) => | 777 | set: (id, value) => |
| @@ -779,7 +780,10 @@ describe("connections.create credential-write compensation", () => { · 2 unmodified lines |
| 779 | : Effect.sync(() => void store.set(String(id), value)), | 780 | : Effect.sync(() => void store.set(String(id), value)), |
| 780 | }); | 781 | }); |
| 781 | const config = makeTestConfig({ plugins: [durabilityPlugin(provider)] as const }); | 782 | const config = makeTestConfig({ plugins: [durabilityPlugin(provider)] as const }); |
| 782 | const executor = yield* createExecutor({ ...config, db: failConnectionDeletes(config.db) }); | 783 | const executor = yield* createExecutor({ |
| | 784 | ...config, |
| | 785 | db: failableConnectionDeletes(config.db, () => failRowDelete), |
| | 786 | }); |
| 783 | yield* executor.durable.seed(); | 787 | yield* executor.durable.seed(); |
| 784 | failRowDelete = true; | 788 | failRowDelete = true; |
| 785 | | 789 | |
| @@ -808,13 +812,208 @@ describe("connections.create credential-write compensation", () => { · 22 unmodified lines |
| 808 | expect(failure.cause.message).toBe("provider write refused"); | 812 | expect(failure.cause.message).toBe("provider write refused"); |
| 809 | | 813 | |
| 810 | // Non-vacuous: the compensating delete really did fail, so the | 814 | // Non-vacuous: the compensating delete really did fail, so the |
| 811 | // credential-less row the error names is still there. | 815 | // row the error names is still there. |
| | 816 | failRowDelete = false; |
| | 817 | const rows = yield* executor.connections.list(); |
| | 818 | expect(rows.length).toBe(1); |
| | 819 | expect(String(rows[0]?.name)).toBe("main"); |
| | 820 | }), |
| | 821 | ); |
| | 822 | |
| | 823 | // Compensation can be slow (provider calls). In that window a concurrent |
| | 824 | // remove can free the name and a new create can take it, writing fresh |
| | 825 | // secrets at the SAME deterministic item ids. Late compensation must then |
| | 826 | // recognize that the row is no longer the one it inserted — identified by |
| | 827 | // the storage surrogate row id — and touch neither the replacement row nor |
| | 828 | // its credentials. Losing compensation to a concurrent remove is correct: |
| | 829 | // the remover already cleaned up. |
| | 830 | it.effect("late compensation leaves a concurrent replacement untouched", () => |
| | 831 | Effect.scoped( |
| | 832 | Effect.gen(function* () { |
| | 833 | const secondWriteEntered = yield* Deferred.make<void>(); |
| | 834 | const releaseSecondWrite = yield* Deferred.make<void>(); |
| | 835 | const store = new Map<string, string>(); |
| | 836 | let parkNextSecondWrite = true; |
| | 837 | const provider = trackingProvider(store, { |
| | 838 | set: (id, value) => { |
| | 839 | if (String(id).endsWith(":second") && parkNextSecondWrite) { |
| | 840 | parkNextSecondWrite = false; |
| | 841 | return Deferred.succeed(secondWriteEntered, undefined).pipe( |
| | 842 | Effect.andThen(Deferred.await(releaseSecondWrite)), |
| | 843 | Effect.andThen( |
| | 844 | Effect.fail( |
| | 845 | new StorageError({ message: "provider write refused", cause: undefined }), |
| | 846 | ), |
| | 847 | ), |
| | 848 | ); |
| | 849 | } |
| | 850 | return Effect.sync(() => void store.set(String(id), value)); |
| | 851 | }, |
| | 852 | }); |
| | 853 | const executor = yield* makeTestExecutor({ |
| | 854 | plugins: [durabilityPlugin(provider)] as const, |
| | 855 | }); |
| | 856 | yield* executor.durable.seed(); |
| | 857 | |
| | 858 | const fiber = yield* Effect.forkChild( |
| | 859 | executor.connections.create({ |
| | 860 | owner: "org", |
| | 861 | name: ConnectionName.make("main"), |
| | 862 | integration: INTEG, |
| | 863 | template: TEMPLATE, |
| | 864 | values: { first: "a-1", second: "a-2" }, |
| | 865 | }), |
| | 866 | ); |
| | 867 | yield* Deferred.await(secondWriteEntered); |
| | 868 | |
| | 869 | // While the first create is parked in its provider write, the user |
| | 870 | // removes the connection and recreates it with different secrets. |
| | 871 | yield* executor.connections.remove({ |
| | 872 | owner: "org", |
| | 873 | integration: INTEG, |
| | 874 | name: ConnectionName.make("main"), |
| | 875 | }); |
| | 876 | yield* executor.connections.create({ |
| | 877 | owner: "org", |
| | 878 | name: ConnectionName.make("main"), |
| | 879 | integration: INTEG, |
| | 880 | template: TEMPLATE, |
| | 881 | values: { first: "c-1", second: "c-2" }, |
| | 882 | }); |
| | 883 | |
| | 884 | // Release the parked write: the first create fails and compensates |
| | 885 | // late, against a name it no longer owns. |
| | 886 | yield* Deferred.succeed(releaseSecondWrite, undefined); |
| | 887 | const exit = yield* Fiber.await(fiber); |
| | 888 | expect(Exit.isFailure(exit)).toBe(true); |
| | 889 | |
| | 890 | // The replacement row AND its credentials survive. |
| | 891 | const rows = yield* executor.connections.list(); |
| | 892 | expect(rows.length).toBe(1); |
| | 893 | expect(String(rows[0]?.name)).toBe("main"); |
| | 894 | expect(store.get("connection:org:vercel:main:first")).toBe("c-1"); |
| | 895 | expect(store.get("connection:org:vercel:main:second")).toBe("c-2"); |
| | 896 | }), |
| | 897 | ), |
| | 898 | ); |
| | 899 | |
| | 900 | // A provider write can die with a defect instead of failing. The stranded- |
| | 901 | // row promise must hold there too: a defect followed by a failed |
| | 902 | // compensating delete surfaces the same typed StorageError naming the |
| | 903 | // stranded connection, not an anonymous crash. |
| | 904 | it.effect("a defect followed by a failed row delete still names the stranded connection", () => |
| | 905 | Effect.gen(function* () { |
| | 906 | let failRowDelete = false; |
| | 907 | const store = new Map<string, string>(); |
| | 908 | const provider = trackingProvider(store, { |
| | 909 | set: (id, value) => |
| | 910 | String(id).endsWith(":second") |
| | 911 | ? Effect.die("provider crashed") |
| | 912 | : Effect.sync(() => void store.set(String(id), value)), |
| | 913 | }); |
| | 914 | const config = makeTestConfig({ plugins: [durabilityPlugin(provider)] as const }); |
| | 915 | const executor = yield* createExecutor({ |
| | 916 | ...config, |
| | 917 | db: failableConnectionDeletes(config.db, () => failRowDelete), |
| | 918 | }); |
| | 919 | yield* executor.durable.seed(); |
| | 920 | failRowDelete = true; |
| | 921 | |
| | 922 | const exit = yield* Effect.exit( |
| | 923 | executor.connections.create({ |
| | 924 | owner: "org", |
| | 925 | name: ConnectionName.make("main"), |
| | 926 | integration: INTEG, |
| | 927 | template: TEMPLATE, |
| | 928 | values: { first: "1", second: "2" }, |
| | 929 | }), |
| | 930 | ); |
| | 931 | |
| | 932 | expect(Exit.isFailure(exit)).toBe(true); |
| | 933 | if (!Exit.isFailure(exit)) return; |
| | 934 | // Not an anonymous defect: the typed error is on the failure channel. |
| | 935 | expect(Cause.hasFails(exit.cause)).toBe(true); |
| | 936 | const failure = Cause.squash(exit.cause); |
| | 937 | const isStorageError = (u: unknown): u is StorageError => |
| | 938 | Predicate.isTagged("StorageError")(u); |
| | 939 | expect(isStorageError(failure)).toBe(true); |
| | 940 | if (!isStorageError(failure)) return; |
| | 941 | expect(failure.message).toContain("main"); |
| | 942 | expect(failure.message).toContain("vercel"); |
| | 943 | expect(failure.message).toContain("stranded"); |
| | 944 | // The original defect is retained as the cause, not replaced. |
| | 945 | expect(failure.cause).toBe("provider crashed"); |
| | 946 | |
| | 947 | // Non-vacuous: the row the error names is still there. |
| 812 | failRowDelete = false; | 948 | failRowDelete = false; |
| 813 | const rows = yield* executor.connections.list(); | 949 | const rows = yield* executor.connections.list(); |
| 814 | expect(rows.length).toBe(1); | 950 | expect(rows.length).toBe(1); |
| 815 | expect(String(rows[0]?.name)).toBe("main"); | 951 | expect(String(rows[0]?.name)).toBe("main"); |
| 816 | }), | 952 | }), |
| 817 | ); | 953 | ); |
| | 954 | |
| | 955 | // Interruption cannot carry a typed error — interrupting wins over failing |
| | 956 | // — so when an interrupted create cannot delete its row, the stranded row |
| | 957 | // is reported through a loud error log and the create stays an |
| | 958 | // interruption. The items that landed before the interrupt stay with the |
| | 959 | // stranded row: credential teardown is gated on the row delete succeeding. |
| | 960 | it.effect("an interrupted create with a failed row delete logs the stranded row", () => |
| | 961 | Effect.scoped( |
| | 962 | Effect.gen(function* () { |
| | 963 | let failRowDelete = false; |
| | 964 | const secondWriteEntered = yield* Deferred.make<void>(); |
| | 965 | const store = new Map<string, string>(); |
| | 966 | const provider = trackingProvider(store, { |
| | 967 | set: (id, value) => |
| | 968 | String(id).endsWith(":second") |
| | 969 | ? Deferred.succeed(secondWriteEntered, undefined).pipe(Effect.andThen(Effect.never)) |
| | 970 | : Effect.sync(() => void store.set(String(id), value)), |
| | 971 | }); |
| | 972 | const config = makeTestConfig({ plugins: [durabilityPlugin(provider)] as const }); |
| | 973 | const executor = yield* createExecutor({ |
| | 974 | ...config, |
| | 975 | db: failableConnectionDeletes(config.db, () => failRowDelete), |
| | 976 | }); |
| | 977 | yield* executor.durable.seed(); |
| | 978 | |
| | 979 | const errors: string[] = []; |
| | 980 | const capture = Logger.make<unknown, void>((options) => { |
| | 981 | if (options.logLevel === "Error") { |
| | 982 | errors.push(Inspectable.toStringUnknown(options.message, 0)); |
| | 983 | } |
| | 984 | }); |
| | 985 | const fiber = yield* Effect.forkChild( |
| | 986 | executor.connections |
| | 987 | .create({ |
| | 988 | owner: "org", |
| | 989 | name: ConnectionName.make("main"), |
| | 990 | integration: INTEG, |
| | 991 | template: TEMPLATE, |
| | 992 | values: { first: "1", second: "2" }, |
| | 993 | }) |
| | 994 | .pipe(Effect.provide(Logger.layer([capture]))), |
| | 995 | ); |
| | 996 | yield* Deferred.await(secondWriteEntered); |
| | 997 | failRowDelete = true; |
| | 998 | yield* Fiber.interrupt(fiber); |
| | 999 | const exit = yield* Fiber.await(fiber); |
| | 1000 | |
| | 1001 | // Still an interruption — and the stranded row was reported loudly. |
| | 1002 | expect(Exit.isFailure(exit)).toBe(true); |
| | 1003 | if (!Exit.isFailure(exit)) return; |
| | 1004 | expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true); |
| | 1005 | expect(errors.some((line) => line.includes("stranded"))).toBe(true); |
| | 1006 | |
| | 1007 | // Non-vacuous: the row survived the failed delete, and the item that |
| | 1008 | // landed before the interrupt stayed with it. |
| | 1009 | failRowDelete = false; |
| | 1010 | const rows = yield* executor.connections.list(); |
| | 1011 | expect(rows.length).toBe(1); |
| | 1012 | expect(String(rows[0]?.name)).toBe("main"); |
| | 1013 | expect(store.size).toBe(1); |
| | 1014 | }), |
| | 1015 | ), |
| | 1016 | ); |
| 818 | }); | 1017 | }); |
| 819 | | 1018 | |
| 820 | describe("connections.list / get", () => { | 1019 | describe("connections.list / get", () => { |