clean up sources

Rhys Sullivan committed 7 months ago

Commit c45a85b1 parent 6f6f1b121 files · +25 −138Browse files

Unified · Split

21 files

.executor/executor.jsonc +2 −52
apps/web/src/components/shell.tsx +1 −1
apps/web/src/views/home.tsx +1 −1
examples/sqlite-sdk-consumer/src/sqlite-backend.ts +0 −2
packages/platform/sdk-file/src/capture-release-workspace-fixture.ts +1 −6
packages/platform/sdk-file/src/config.test.ts +2 −8
packages/platform/sdk/src/runtime/catalog/source/runtime.ts +0 −3
packages/platform/sdk/src/runtime/catalog/source/sync.ts +5 −4
packages/platform/sdk/src/runtime/sources/source-definitions.ts +0 −8
packages/platform/sdk/src/runtime/sources/source-plugins/internal.ts +1 −1
packages/platform/sdk/src/runtime/sources/source-store/config.ts +0 −3
packages/platform/sdk/src/runtime/sources/source-store/lifecycle.ts +2 −2
packages/platform/sdk/src/runtime/sources/source-store/records.ts +0 −3
packages/platform/sdk/src/schema/models/local-config.ts +0 −5
packages/platform/sdk/src/schema/models/source.ts +0 −15
packages/platform/sdk/src/sources/operations.ts +0 −1
packages/sources/core/src/catalog-fragment.ts +3 −3
packages/sources/core/src/catalog-shared.ts +4 −6
packages/sources/core/src/source-models.ts +0 −3
plugins/openapi/react/index.tsx +0 −4
plugins/openapi/sdk/index.ts +3 −7

@@ -2,63 +2,13 @@ · 1 unmodified line
2	2	"sources": {
3	3	"github": {
4	4	"kind": "openapi",
5		− "name": "GitHub",
6	5	"connection": {
7		− "endpoint": "https://api.github.com"
	6	+ "endpoint": "https://raw.githubusercontent.com/github/rest-api-description/refs/heads/main/descriptions/ghes-3.9/%7Bprotocol%7D://%7Bhostname%7D/api/v3"
8	7	},
9	8	"binding": {
10		− "specUrl": "https://example.com/github-openapi.yaml",
	9	+ "specUrl": "https://raw.githubusercontent.com/github/rest-api-description/refs/heads/main/descriptions/ghes-3.9/ghes-3.9.yaml",
11	10	"defaultHeaders": null
12	11	}
13		− },
14		− "google-gmail": {
15		− "name": "Gmail",
16		− "namespace": "google.gmail",
17		− "connection": {
18		− "endpoint": "https://www.googleapis.com/discovery/v1/apis/gmail/v1/rest"
19		− },
20		− "kind": "google_discovery",
21		− "binding": {
22		− "service": "gmail",
23		− "version": "v1",
24		− "discoveryUrl": "https://www.googleapis.com/discovery/v1/apis/gmail/v1/rest",
25		− "defaultHeaders": null,
26		− "scopes": [
27		− "scope:gmail.readonly"
28		− ]
29		− }
30		− },
31		− "google-calendar": {
32		− "name": "Google Calendar",
33		− "namespace": "google.calendar",
34		− "connection": {
35		− "endpoint": "https://calendar-json.googleapis.com/$discovery/rest?version=v3"
36		− },
37		− "kind": "google_discovery",
38		− "binding": {
39		− "service": "calendar",
40		− "version": "v3",
41		− "discoveryUrl": "https://calendar-json.googleapis.com/$discovery/rest?version=v3",
42		− "defaultHeaders": null,
43		− "scopes": [
44		− "scope:calendar.readonly"
45		− ]
46		− }
47		− },
48		− "google-drive": {
49		− "name": "Drive",
50		− "namespace": "google.drive",
51		− "connection": {
52		− "endpoint": "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest"
53		− },
54		− "kind": "google_discovery",
55		− "binding": {
56		− "service": "drive",
57		− "version": "v3",
58		− "discoveryUrl": "https://www.googleapis.com/discovery/v1/apis/drive/v3/rest",
59		− "defaultHeaders": null,
60		− "scopes": []
61		− }
62	12	}
63	13	}
64	14	}

apps/web/src/components/shell.tsx+1−1

@@ -331,7 +331,7 @@ function SourceItem(props: { · 330 unmodified lines
331	331	)}
332	332	>
333	333	<div className="flex size-3 shrink-0 items-center justify-center text-muted-foreground/50">
334		− <SourceFavicon endpoint={source.endpoint} kind={source.kind} className="size-3" />
	334	+ <SourceFavicon kind={source.kind} className="size-3" />
335	335	</div>
336	336	<span className="flex-1 truncate">{source.name}</span>
337	337	<span

apps/web/src/views/home.tsx+1−1

@@ -58,7 +58,7 @@ export function HomePage() { · 57 unmodified lines
58	58	{source.name}
59	59	</div>
60	60	<div className="mt-1 text-xs text-muted-foreground">
61		− {source.endpoint}
	61	+ {source.kind}
62	62	</div>
63	63	</div>
64	64	<Badge

examples/sqlite-sdk-consumer/src/sqlite-backend.ts+0−2

@@ -127,11 +127,9 @@ const createSourceArtifact = (input: SourceArtifactBuildInput): SourceArtifact = · 126 unmodified lines
127	127	const snapshot = snapshotFromSourceCatalogSyncResult(input.syncResult);
128	128	const sourceConfigJson = JSON.stringify({
129	129	kind: input.source.kind,
130		− endpoint: input.source.endpoint,
131	130	namespace: input.source.namespace,
132	131	name: input.source.name,
133	132	enabled: input.source.enabled,
134		− sourceHash: input.source.sourceHash,
135	133	});
136	134	const importMetadataJson = JSON.stringify(snapshot.import);
137	135	const catalogId = SourceCatalogIdSchema.make(`src_catalog_${makeHash(sourceConfigJson)}`);

packages/platform/sdk-file/src/capture-release-workspace-fixture.ts+1−6

@@ -187,12 +187,7 @@ const copyRecursive = ( · 186 unmodified lines
187	187
188	188	const sanitizeSourceConfig = (
189	189	sourceConfig: NonNullable<LocalExecutorConfig["sources"]>[string],
190		−): NonNullable<LocalExecutorConfig["sources"]>[string] => {
191		− return {
192		− ...sourceConfig,
193		− connection: sourceConfig.connection,
194		− };
195		−};
	190	+): NonNullable<LocalExecutorConfig["sources"]>[string] => sourceConfig;
196	191
197	192	const main = Effect.gen(function* () {
198	193	const args = yield* Effect.try({

packages/platform/sdk-file/src/config.test.ts+2−8

@@ -38,9 +38,6 @@ describe("local-config", () => { · 37 unmodified lines
38	38	"sources": {
39	39	"github": {
40	40	"kind": "openapi",
41		− "connection": {
42		− "endpoint": "https://api.github.com",
43		− },
44	41	"binding": {
45	42	"specUrl": "https://example.com/openapi.json",
46	43	},
@@ -54,9 +51,6 @@ describe("local-config", () => { · 7 unmodified lines
54	51	const loaded = yield* loadLocalExecutorConfig(context);
55	52
56	53	expect(loaded.config?.sources?.github?.kind).toBe("openapi");
57		− expect(loaded.config?.sources?.github?.connection.endpoint).toBe(
58		− "https://api.github.com",
59		− );
60	54	expect(loaded.config?.runtime).toBe("ses");
61	55	expect(context.homeStateDirectory.toLowerCase()).toContain("executor");
62	56	}).pipe(Effect.provide(NodeFileSystem.layer)),
@@ -74,8 +68,8 @@ describe("local-config", () => { · 11 unmodified lines
74	68	"sources": {
75	69	"github": {
76	70	"kind": "openapi"
77		− "connection": {
78		− "endpoint": "https://api.github.com"
	71	+ "binding": {
	72	+ "specUrl": "https://api.github.com/openapi.json"
79	73	}
80	74	}
81	75	}

packages/platform/sdk/src/runtime/catalog/source/runtime.ts+0−3

@@ -675,12 +675,9 @@ const sourceRecordFromCatalogArtifact = (input: { · 674 unmodified lines
675	675	catalogRevisionId: input.artifact.revision.id,
676	676	name: input.source.name,
677	677	kind: input.source.kind,
678		− endpoint: input.source.endpoint,
679	678	status: input.source.status,
680	679	enabled: input.source.enabled,
681	680	namespace: input.source.namespace,
682		− sourceHash: input.source.sourceHash,
683		− lastError: input.source.lastError,
684	681	createdAt: input.source.createdAt,
685	682	updatedAt: input.source.updatedAt,
686	683	});

packages/platform/sdk/src/runtime/catalog/source/sync.ts+5−4

@@ -89,7 +89,8 @@ const syncSourceCatalogWithDeps = ( · 88 unmodified lines
89	89
90	90	if (!shouldIndexSource(input.source)) {
91	91	const state = yield* deps.scopeStateStore.load();
92		− const existingSourceState = state.sources[input.source.id];
	92	+ const existingSourceState =
	93	+ state.sources[input.source.id] as LocalScopeState["sources"][string] \| undefined;
93	94	const nextState: LocalScopeState = {
94	95	...state,
95	96	sources: {
@@ -97,7 +98,7 @@ const syncSourceCatalogWithDeps = ( · 1 unmodified line
97	98	[input.source.id]: {
98	99	status: (input.source.enabled ? input.source.status : "draft") as SourceStatus,
99	100	lastError: null,
100		− sourceHash: input.source.sourceHash,
	101	+ sourceHash: existingSourceState?.sourceHash ?? null,
101	102	createdAt: existingSourceState?.createdAt ?? input.source.createdAt,
102	103	updatedAt: Date.now(),
103	104	},
@@ -127,7 +128,8 @@ const syncSourceCatalogWithDeps = ( · 23 unmodified lines
127	128	});
128	129
129	130	const state = yield* deps.scopeStateStore.load();
130		− const existingSourceState = state.sources[input.source.id];
	131	+ const existingSourceState =
	132	+ state.sources[input.source.id] as LocalScopeState["sources"][string] \| undefined;
131	133	const nextState: LocalScopeState = {
132	134	...state,
133	135	sources: {
@@ -155,7 +157,6 @@ const syncSourceCatalogWithDeps = ( · 21 unmodified lines
155	157	"executor.source.id": input.source.id,
156	158	"executor.source.kind": input.source.kind,
157	159	"executor.source.namespace": input.source.namespace,
158		− "executor.source.endpoint": input.source.endpoint,
159	160	},
160	161	}),
161	162	);

packages/platform/sdk/src/runtime/sources/source-definitions.ts+0−8

@@ -34,7 +34,6 @@ const sourceConfigFromSource = (source: Source): SourceCatalogSourceConfig => · 33 unmodified lines
34	34	source,
35	35	}) ?? {
36	36	kind: source.kind,
37		− endpoint: source.endpoint,
38	37	namespace: source.namespace,
39	38	};
40	39
@@ -60,7 +59,6 @@ export const sourceConfigSignature = (source: Source): string => · 19 unmodified lines
60	59	sourceId: source.id,
61	60	kind: source.kind,
62	61	name: source.name,
63		− endpoint: source.endpoint,
64	62	namespace: source.namespace,
65	63	enabled: source.enabled,
66	64	updatedAt: source.updatedAt,
@@ -91,12 +89,9 @@ export const normalizeSourceForCreate = (input: { · 24 unmodified lines
91	89	scopeId: input.scopeId,
92	90	name: input.source.name.trim(),
93	91	kind: input.source.kind,
94		− endpoint: input.source.endpoint.trim(),
95	92	status: input.source.status,
96	93	enabled: input.source.enabled,
97	94	namespace: trimOrNull(input.source.namespace),
98		− sourceHash: trimOrNull(input.source.sourceHash),
99		− lastError: trimOrNull(input.source.lastError),
100	95	createdAt: input.now,
101	96	updatedAt: input.now,
102	97	});
@@ -108,10 +103,7 @@ export const normalizeSourceForSave = (input: { · 5 unmodified lines
108	103	validateSourceByKind({
109	104	...input.source,
110	105	name: input.source.name.trim(),
111		− endpoint: input.source.endpoint.trim(),
112	106	namespace: trimOrNull(input.source.namespace),
113		− sourceHash: trimOrNull(input.source.sourceHash),
114		− lastError: trimOrNull(input.source.lastError),
115	107	updatedAt: input.now,
116	108	});
117	109

packages/platform/sdk/src/runtime/sources/source-plugins/internal.ts+1−1

@@ -17,7 +17,7 @@ export const InternalSourceSdkPlugin = { · 16 unmodified lines
17	17	catalogKind: "internal",
18	18	catalogIdentity: ({ source }) => ({
19	19	kind: "internal",
20		− endpoint: source.endpoint,
	20	+ sourceId: source.id,
21	21	}),
22	22	getIrModel: ({ source }) =>
23	23	Effect.succeed(

packages/platform/sdk/src/runtime/sources/source-store/config.ts+0−3

@@ -50,8 +50,5 @@ export const configSourceFromLocalSource = (input: { · 49 unmodified lines
50	50	? { namespace: input.source.namespace ?? undefined }
51	51	: {}),
52	52	...(input.source.enabled === false ? { enabled: false } : {}),
53		− connection: {
54		− endpoint: input.source.endpoint,
55		− },
56	53	} as LocalConfigSource;
57	54	};

packages/platform/sdk/src/runtime/sources/source-store/lifecycle.ts+2−2

@@ -110,8 +110,8 @@ export const persistSourceWithDeps = ( · 109 unmodified lines
110	110	...localScope.scopeState.sources,
111	111	[nextSource.id]: {
112	112	status: nextSource.status,
113		− lastError: nextSource.lastError,
114		− sourceHash: nextSource.sourceHash,
	113	+ lastError: existingSourceState?.lastError ?? null,
	114	+ sourceHash: existingSourceState?.sourceHash ?? null,
115	115	createdAt: existingSourceState?.createdAt ?? nextSource.createdAt,
116	116	updatedAt: nextSource.updatedAt,
117	117	},

packages/platform/sdk/src/runtime/sources/source-store/records.ts+0−3

@@ -47,14 +47,11 @@ export const buildLocalSourceRecord = (input: { · 46 unmodified lines
47	47	scopeId: input.scopeId,
48	48	name: trimOrNull(sourceConfig.name) ?? input.sourceId,
49	49	kind: sourceConfig.kind,
50		− endpoint: sourceConfig.connection.endpoint.trim(),
51	50	status:
52	51	existingState?.status ??
53	52	(sourceConfig.enabled ?? true ? "connected" : "draft"),
54	53	enabled: sourceConfig.enabled ?? true,
55	54	namespace: trimOrNull(sourceConfig.namespace) ?? input.sourceId,
56		− sourceHash: existingState?.sourceHash ?? null,
57		− lastError: existingState?.lastError ?? null,
58	55	createdAt: existingState?.createdAt ?? Date.now(),
59	56	updatedAt: existingState?.updatedAt ?? Date.now(),
60	57	};

packages/platform/sdk/src/schema/models/local-config.ts+0−5

@@ -56,15 +56,10 @@ export const LocalConfigSecretInputSchema = Schema.Union( · 55 unmodified lines
56	56	LocalConfigExplicitSecretRefSchema,
57	57	);
58	58
59		−export const LocalConfigSourceConnectionSchema = Schema.Struct({
60		− endpoint: Schema.String,
61		−});
62		−
63	59	const LocalConfigSourceEntryBaseSchema = Schema.Struct({
64	60	name: Schema.optional(Schema.String),
65	61	namespace: Schema.optional(Schema.String),
66	62	enabled: Schema.optional(Schema.Boolean),
67		− connection: LocalConfigSourceConnectionSchema,
68	63	});
69	64
70	65	export const LocalConfigSourceSchema = Schema.extend(

packages/platform/sdk/src/schema/models/source.ts+0−15

@@ -36,12 +36,9 @@ const SourceStorageRowSchema = Schema.Struct({ · 35 unmodified lines
36	36	catalogRevisionId: SourceCatalogRevisionIdSchema,
37	37	name: Schema.String,
38	38	kind: SourceKindSchema,
39		− endpoint: Schema.String,
40	39	status: SourceStatusSchema,
41	40	enabled: Schema.Boolean,
42	41	namespace: Schema.NullOr(Schema.String),
43		− sourceHash: Schema.NullOr(Schema.String),
44		− lastError: Schema.NullOr(Schema.String),
45	42	createdAt: TimestampMsSchema,
46	43	updatedAt: TimestampMsSchema,
47	44	});
@@ -55,12 +52,9 @@ export const StoredSourceRecordSchema = Schema.transform( · 7 unmodified lines
55	52	catalogRevisionId: SourceCatalogRevisionIdSchema,
56	53	name: Schema.String,
57	54	kind: SourceKindSchema,
58		− endpoint: Schema.String,
59	55	status: SourceStatusSchema,
60	56	enabled: Schema.Boolean,
61	57	namespace: Schema.NullOr(Schema.String),
62		− sourceHash: Schema.NullOr(Schema.String),
63		− lastError: Schema.NullOr(Schema.String),
64	58	createdAt: TimestampMsSchema,
65	59	updatedAt: TimestampMsSchema,
66	60	}),
@@ -73,12 +67,9 @@ export const StoredSourceRecordSchema = Schema.transform( · 6 unmodified lines
73	67	catalogRevisionId: row.catalogRevisionId,
74	68	name: row.name,
75	69	kind: row.kind,
76		− endpoint: row.endpoint,
77	70	status: row.status,
78	71	enabled: row.enabled,
79	72	namespace: row.namespace,
80		− sourceHash: row.sourceHash,
81		− lastError: row.lastError,
82	73	createdAt: row.createdAt,
83	74	updatedAt: row.updatedAt,
84	75	}),
@@ -89,12 +80,9 @@ export const StoredSourceRecordSchema = Schema.transform( · 4 unmodified lines
89	80	catalogRevisionId: source.catalogRevisionId,
90	81	name: source.name,
91	82	kind: source.kind,
92		− endpoint: source.endpoint,
93	83	status: source.status,
94	84	enabled: source.enabled,
95	85	namespace: source.namespace,
96		− sourceHash: source.sourceHash,
97		− lastError: source.lastError,
98	86	createdAt: source.createdAt,
99	87	updatedAt: source.updatedAt,
100	88	}),
@@ -106,12 +94,9 @@ export const SourceSchema = Schema.Struct({ · 5 unmodified lines
106	94	scopeId: ScopeIdSchema,
107	95	name: Schema.String,
108	96	kind: SourceKindSchema,
109		− endpoint: Schema.String,
110	97	status: SourceStatusSchema,
111	98	enabled: Schema.Boolean,
112	99	namespace: Schema.NullOr(Schema.String),
113		− sourceHash: Schema.NullOr(Schema.String),
114		− lastError: Schema.NullOr(Schema.String),
115	100	createdAt: TimestampMsSchema,
116	101	updatedAt: TimestampMsSchema,
117	102	});

packages/platform/sdk/src/sources/operations.ts+0−1

@@ -62,7 +62,6 @@ const syncArtifactsForSource = (input: { · 61 unmodified lines
62	62	source: {
63	63	...input.source,
64	64	status: "error",
65		− lastError: error.message,
66	65	},
67	66	now: Date.now(),
68	67	}).pipe(

packages/sources/core/src/catalog-fragment.ts+3−3

@@ -117,7 +117,7 @@ const createServiceScope = (input: { · 116 unmodified lines
117	117	export const buildCatalogFragment = (input: {
118	118	source: Pick<
119	119	Source,
120		− "id" \| "kind" \| "name" \| "endpoint" \| "namespace"
	120	+ "id" \| "kind" \| "name" \| "namespace"
121	121	>;
122	122	documents: readonly CatalogSourceDocumentInput[];
123	123	serviceScopeDefaults?: Scope["defaults"];
@@ -131,14 +131,14 @@ export const buildCatalogFragment = (input: { · 7 unmodified lines
131	131	: [
132	132	{
133	133	documentKind: "synthetic",
134		− documentKey: input.source.endpoint,
	134	+ documentKey: input.source.id,
135	135	fetchedAt: Date.now(),
136	136	contentText: "{}",
137	137	},
138	138	];
139	139	const primaryDocument = documents[0]!;
140	140	const primaryDocumentKey =
141		− primaryDocument.documentKey ?? input.source.endpoint ?? input.source.id;
	141	+ primaryDocument.documentKey ?? input.source.id;
142	142	const primaryDocumentId = documentIdFor(
143	143	input.source,
144	144	`${primaryDocument.documentKind}:${primaryDocument.documentKey}`,

packages/sources/core/src/catalog-shared.ts+4−6

@@ -120,18 +120,16 @@ export const interactionForEffect = ( · 119 unmodified lines
120	120	});
121	121
122	122	export const createCatalogImportMetadata = (input: {
123		− source: Pick<Source, "kind" \| "endpoint" \| "sourceHash">;
	123	+ source: Pick<Source, "id" \| "kind">;
124	124	pluginKey: string;
125	125	}): ImportMetadata => ({
126	126	sourceKind: sourceKindFromSource(input.source),
127	127	pluginKey: input.pluginKey,
128	128	importerVersion: "ir.v1.snapshot_builder",
129	129	importedAt: new Date().toISOString(),
130		− sourceConfigHash:
131		− input.source.sourceHash ??
132		− stableHash({
133		− endpoint: input.source.endpoint,
134		− }),
	130	+ sourceConfigHash: stableHash({
	131	+ sourceId: input.source.id,
	132	+ }),
135	133	});
136	134
137	135	export const docsFrom = (input: {

packages/sources/core/src/source-models.ts+0−3

@@ -73,12 +73,9 @@ export const SourceSchema = Schema.Struct({ · 72 unmodified lines
73	73	scopeId: Schema.String,
74	74	name: Schema.String,
75	75	kind: SourceKindSchema,
76		− endpoint: Schema.String,
77	76	status: SourceStatusSchema,
78	77	enabled: Schema.Boolean,
79	78	namespace: Schema.NullOr(Schema.String),
80		− sourceHash: Schema.NullOr(Schema.String),
81		− lastError: Schema.NullOr(Schema.String),
82	79	createdAt: Schema.Number,
83	80	updatedAt: Schema.Number,
84	81	});

plugins/openapi/react/index.tsx+0−4

@@ -400,8 +400,6 @@ function OpenApiEditSourcePage(props: { · 399 unmodified lines
400	400	{JSON.stringify({
401	401	id: props.source.id,
402	402	kind: props.source.kind,
403		− endpoint: props.source.endpoint,
404		− sourceHash: props.source.sourceHash,
405	403	}, null, 2)}
406	404	</pre>
407	405	</Section>
@@ -423,9 +421,7 @@ function OpenApiSourceDetailPage(props: { · 15 unmodified lines
423	421	id: props.source.id,
424	422	name: props.source.name,
425	423	kind: props.source.kind,
426		− endpoint: props.source.endpoint,
427	424	status: props.source.status,
428		− sourceHash: props.source.sourceHash,
429	425	}, null, 2)}
430	426	</pre>
431	427	</Section>

plugins/openapi/sdk/index.ts+3−7

@@ -76,7 +76,7 @@ const createOpenApiSourceRuntime = ( · 75 unmodified lines
76	76	catalogKind: "imported",
77	77	catalogIdentity: ({ source }) => ({
78	78	kind: "openapi",
79		− endpoint: source.endpoint,
	79	+ sourceId: source.id,
80	80	}),
81	81	getIrModel: ({ source }) =>
82	82	Effect.gen(function* () {
@@ -96,10 +96,9 @@ const createOpenApiSourceRuntime = ( · 13 unmodified lines
96	96	}),
97	97	importerVersion: "ir.v1.openapi",
98	98	sourceConfigHash:
99		− stored === null ? source.sourceHash ?? "openapi" : stableSourceHash(stored),
	99	+ stored === null ? "openapi" : stableSourceHash(stored),
100	100	},
101		− sourceHash:
102		− stored === null ? source.sourceHash ?? null : stableSourceHash(stored),
	101	+ sourceHash: stored === null ? null : stableSourceHash(stored),
103	102	});
104	103	}),
105	104	invoke: () =>
@@ -125,12 +124,9 @@ export const openApiSdkPlugin = ( · 19 unmodified lines
125	124	source: {
126	125	name: input.name.trim(),
127	126	kind: "openapi",
128		− endpoint: stored.baseUrl ?? stored.specUrl,
129	127	status: "connected",
130	128	enabled: true,
131	129	namespace: null,
132		− sourceHash: stableSourceHash(stored),
133		− lastError: null,
134	130	},
135	131	});
136	132