History

fix(git): retry failed upstream refreshes

Kevin Beier committed 3 months ago

Commit 375ea951 parent 399b0932 files · +57 −1Browse files

Unified · Split

2 files
  1. apps/server/src/vcs/VcsStatusBroadcaster.test.ts +56 −0
  2. apps/server/src/vcs/VcsStatusBroadcaster.ts +1 −1
apps/server/src/vcs/VcsStatusBroadcaster.test.ts+56−0
@@ -652,6 +652,62 @@ describe("VcsStatusBroadcaster", () => { · 651 unmodified lines
652 }).pipe(Effect.provide(Layer.merge(makeTestLayer(state), TestClock.layer())));652 }).pipe(Effect.provide(Layer.merge(makeTestLayer(state), TestClock.layer())));
653 });653 });
654 654
655 it.effect("retries a failed upstream refresh after the failure backoff", () => {
656 const refreshUpstreamValues: Array<boolean | undefined> = [];
657 let failedUpstreamRefresh = false;
658 const testLayer = VcsStatusBroadcaster.layer.pipe(
659 Layer.provideMerge(NodeServices.layer),
660 Layer.provide(
661 Layer.mock(GitWorkflowService.GitWorkflowService)({
662 localStatus: () => Effect.succeed(baseLocalStatus),
663 remoteStatus: (_input, options) => {
664 refreshUpstreamValues.push(options?.refreshUpstream);
665 if (options?.refreshUpstream && !failedUpstreamRefresh) {
666 failedUpstreamRefresh = true;
667 return Effect.fail(
668 new GitManagerError({
669 operation: "VcsStatusBroadcaster.test",
670 cwd: "/repo",
671 detail: "upstream refresh failure",
672 }),
673 );
674 }
675 return Effect.succeed(baseRemoteStatus);
676 },
677 invalidateRemoteStatus: () => Effect.void,
678 }),
679 ),
680 );
681
682 return Effect.gen(function* () {
683 const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster;
684 yield* broadcaster.getStatus({ cwd: "/repo" });
685 const scope = yield* Scope.make();
686 const snapshotDeferred = yield* Deferred.make<void>();
687 yield* Stream.runForEach(
688 broadcaster.streamStatus(
689 { cwd: "/repo" },
690 { automaticRemoteRefreshInterval: Effect.succeed(Duration.minutes(5)) },
691 ),
692 (event) =>
693 event._tag === "snapshot"
694 ? Deferred.succeed(snapshotDeferred, undefined).pipe(Effect.ignore)
695 : Effect.void,
696 ).pipe(Effect.forkIn(scope));
697 yield* Deferred.await(snapshotDeferred);
698 yield* Effect.yieldNow;
699
700 yield* TestClock.adjust(Duration.minutes(5));
701 assert.isTrue(failedUpstreamRefresh);
702 assert.equal(refreshUpstreamValues.at(-1), true);
703
704 yield* TestClock.adjust(Duration.seconds(30));
705 assert.deepStrictEqual(refreshUpstreamValues.slice(-2), [true, true]);
706
707 yield* Scope.close(scope, Exit.void);
708 }).pipe(Effect.provide(Layer.merge(testLayer, TestClock.layer())));
709 });
710
655 it("backs off remote refresh failures exponentially and honors larger configured intervals", () => {711 it("backs off remote refresh failures exponentially and honors larger configured intervals", () => {
656 assert.equal(712 assert.equal(
657 Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(1, Duration.seconds(1))),713 Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(1, Duration.seconds(1))),
apps/server/src/vcs/VcsStatusBroadcaster.ts+1−1
@@ -509,8 +509,8 @@ export const make = Effect.gen(function* () { · 508 unmodified lines
509 refreshUpstream,509 refreshUpstream,
510 }).pipe(Effect.exit);510 }).pipe(Effect.exit);
511 yield* Ref.set(lastLightweightRefreshRef, refreshTime);511 yield* Ref.set(lastLightweightRefreshRef, refreshTime);
512 if (refreshUpstream) yield* Ref.set(lastUpstreamRefreshRef, refreshTime);
513 if (Exit.isSuccess(exit)) {512 if (Exit.isSuccess(exit)) {
513 if (refreshUpstream) yield* Ref.set(lastUpstreamRefreshRef, refreshTime);
514 yield* Ref.set(needsInitialRefreshRef, false);514 yield* Ref.set(needsInitialRefreshRef, false);
515 yield* Ref.set(consecutiveFailuresRef, 0);515 yield* Ref.set(consecutiveFailuresRef, 0);
516 return;516 return;