import { FileFinder, type MixedItem, type MixedSearchResult } from "@ff-labs/fff-node";import * as Context from "effect/Context";import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";import * as LayerMap from "effect/LayerMap";import * as Schedule from "effect/Schedule";import * as Schema from "effect/Schema"; import type { ProjectEntry, ProjectListEntriesResult, ProjectSearchEntriesResult,} from "@t3tools/contracts"; const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2;const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds";const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";const WORKSPACE_INDEX_SCAN_POLL_INTERVAL = "50 millis"; export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexCreateFailed>()( "WorkspaceSearchIndexCreateFailed", { cwd: Schema.String, reason: Schema.String, cause: Schema.optional(Schema.Defect()), },) { override get message(): string { return `Failed to create the workspace search index for '${this.cwd}'.`; }} export class WorkspaceSearchIndexScanTimedOut extends Schema.TaggedErrorClass<WorkspaceSearchIndexScanTimedOut>()( "WorkspaceSearchIndexScanTimedOut", { cwd: Schema.String, timeout: Schema.String, },) { override get message(): string { return `Workspace search index for '${this.cwd}' did not finish scanning within ${this.timeout}`; }} export class WorkspaceSearchIndexSearchFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexSearchFailed>()( "WorkspaceSearchIndexSearchFailed", { cwd: Schema.String, queryLength: Schema.Number, pageSize: Schema.Number, reason: Schema.String, cause: Schema.optional(Schema.Defect()), },) { override get message(): string { return `Workspace search failed for '${this.cwd}'.`; }} export class WorkspaceSearchIndexRefreshFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexRefreshFailed>()( "WorkspaceSearchIndexRefreshFailed", { cwd: Schema.String, reason: Schema.String, cause: Schema.optional(Schema.Defect()), },) { override get message(): string { return `Failed to refresh the workspace search index for '${this.cwd}'.`; }} export class WorkspaceSearchIndexDestroyFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexDestroyFailed>()( "WorkspaceSearchIndexDestroyFailed", { cwd: Schema.String, cause: Schema.Defect(), },) { override get message(): string { return `Failed to destroy the workspace search index for '${this.cwd}'.`; }} export type WorkspaceSearchIndexError = | WorkspaceSearchIndexCreateFailed | WorkspaceSearchIndexScanTimedOut | WorkspaceSearchIndexSearchFailed | WorkspaceSearchIndexRefreshFailed; export class WorkspaceSearchIndex extends Context.Service< WorkspaceSearchIndex, { readonly list: () => Effect.Effect<ProjectListEntriesResult, WorkspaceSearchIndexSearchFailed>; readonly search: ( query: string, limit: number, ) => Effect.Effect<ProjectSearchEntriesResult, WorkspaceSearchIndexSearchFailed>; readonly refresh: () => Effect.Effect< void, WorkspaceSearchIndexRefreshFailed | WorkspaceSearchIndexScanTimedOut >; }>()("t3/workspace/WorkspaceSearchIndex") {} function toPosixPath(input: string): string { return input.replaceAll("\\", "/");} function trimDirectorySeparator(input: string): string { return input.endsWith("/") ? input.slice(0, -1) : input;} function parentPathOf(input: string): string | undefined { const separatorIndex = input.lastIndexOf("/"); return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex);} function toProjectEntry(item: MixedItem): ProjectEntry | null { const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath)); if (!normalizedPath) { return null; } return { path: normalizedPath, kind: item.type, };} function mapMixedSearchResult( result: MixedSearchResult, limit: number,): { readonly entries: ProjectEntry[]; readonly truncated: boolean } { const entries: ProjectEntry[] = []; for (const item of result.items) { const entry = toProjectEntry(item); if (entry) { entries.push(entry); } if (entries.length >= limit) { break; } } const rootDirectoryCount = result.items.some( (item) => item.type === "directory" && item.item.relativePath.length === 0, ) ? 1 : 0; return { entries, truncated: result.totalMatched - rootDirectoryCount > limit, };} function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEntry[] { const entryByPath = new Map(entries.map((entry) => [entry.path, entry])); for (const entry of entries) { let parentPath = parentPathOf(entry.path); while (parentPath) { if (!entryByPath.has(parentPath)) { entryByPath.set(parentPath, { path: parentPath, kind: "directory" }); } parentPath = parentPathOf(parentPath); } } return [...entryByPath.values()];} const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* (cwd: string) { const result = yield* Effect.try({ try: () => FileFinder.create({ basePath: cwd, disableMmapCache: true, disableContentIndexing: true, aiMode: false, enableFsRootScanning: true, enableHomeDirScanning: true, }), catch: (cause) => new WorkspaceSearchIndexCreateFailed({ cwd, reason: "FileFinder.create threw unexpectedly.", cause, }), }); if (result.ok) return result.value; return yield* new WorkspaceSearchIndexCreateFailed({ cwd, reason: result.error, });}); const waitForScan = <E>(cwd: string, finder: FileFinder, onFailure: (cause: unknown) => E) => Effect.try({ try: () => finder.isScanning(), catch: onFailure, }).pipe( Effect.repeat({ while: (scanning) => scanning, schedule: Schedule.spaced(WORKSPACE_INDEX_SCAN_POLL_INTERVAL), }), Effect.timeoutOrElse({ duration: WORKSPACE_INDEX_SCAN_TIMEOUT, orElse: () => new WorkspaceSearchIndexScanTimedOut({ cwd, timeout: WORKSPACE_INDEX_SCAN_TIMEOUT }), }), Effect.withSpan("WorkspaceSearchIndex.waitForScan"), ); export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (cwd: string) { const finder = yield* Effect.acquireRelease(createFinder(cwd), (finder) => Effect.try({ try: () => finder.destroy(), catch: (cause) => new WorkspaceSearchIndexDestroyFailed({ cwd, cause }), }).pipe(Effect.orDie), ); yield* waitForScan( cwd, finder, (cause) => new WorkspaceSearchIndexCreateFailed({ cwd, reason: "FileFinder.isScanning threw while creating the index.", cause, }), ); const runMixedSearch = Effect.fn("WorkspaceSearchIndex.runMixedSearch")(function* ( query: string, pageSize: number, ) { const result = yield* Effect.try({ try: () => finder.mixedSearch(query, { pageSize }), catch: (cause) => new WorkspaceSearchIndexSearchFailed({ cwd, queryLength: query.length, pageSize, reason: "FileFinder.mixedSearch threw unexpectedly.", cause, }), }); if (!result.ok) { return yield* new WorkspaceSearchIndexSearchFailed({ cwd, queryLength: query.length, pageSize, reason: result.error, }); } return result.value; }); const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn( "WorkspaceSearchIndex.refresh", )(function* () { const result = yield* Effect.try({ try: () => finder.scanFiles(), catch: (cause) => new WorkspaceSearchIndexRefreshFailed({ cwd, reason: "FileFinder.scanFiles threw unexpectedly.", cause, }), }); if (!result.ok) { return yield* new WorkspaceSearchIndexRefreshFailed({ cwd, reason: result.error, }); } yield* waitForScan( cwd, finder, (cause) => new WorkspaceSearchIndexRefreshFailed({ cwd, reason: "FileFinder.isScanning threw while refreshing the index.", cause, }), ); }); const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")( function* () { const result = yield* runMixedSearch("", WORKSPACE_INDEX_PAGE_SIZE); const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES); const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) => left.path.localeCompare(right.path), ); const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES); return { entries, truncated: mapped.truncated || entries.length < sortedEntries.length, }; }, ); const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( "WorkspaceSearchIndex.search", )(function* (query, limit) { const result = yield* runMixedSearch(query, Math.max(1, limit + 1)); return mapMixedSearchResult(result, limit); }); return WorkspaceSearchIndex.of({ list, refresh, search });}); /** * A layer factory is required because every index is scoped to a concrete * workspace root. WorkspaceSearchIndexMap owns memoization and idle cleanup; * using a default cwd here would mix resources from different workspaces. */export const layer = (cwd: string) => Layer.effect(WorkspaceSearchIndex, make(cwd)); export class WorkspaceSearchIndexMap extends LayerMap.Service<WorkspaceSearchIndexMap>()( "t3/workspace/WorkspaceSearchIndexMap", { lookup: layer, idleTimeToLive: WORKSPACE_INDEX_IDLE_TTL, },) {}
import { FileFinder, type MixedItem, type MixedSearchResult } from "@ff-labs/fff-node";
import * as Context from "effect/Context";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as LayerMap from "effect/LayerMap";
import * as Schedule from "effect/Schedule";
import * as Schema from "effect/Schema";
import type {
ProjectEntry,
ProjectListEntriesResult,
ProjectSearchEntriesResult,
} from "@t3tools/contracts";
const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;
const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2;
const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds";
const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
const WORKSPACE_INDEX_SCAN_POLL_INTERVAL = "50 millis";
export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexCreateFailed>()(
"WorkspaceSearchIndexCreateFailed",
{
cwd: Schema.String,
reason: Schema.String,
cause: Schema.optional(Schema.Defect()),
},
) {
override get message(): string {
return `Failed to create the workspace search index for '${this.cwd}'.`;
}
}
export class WorkspaceSearchIndexScanTimedOut extends Schema.TaggedErrorClass<WorkspaceSearchIndexScanTimedOut>()(
"WorkspaceSearchIndexScanTimedOut",
{
cwd: Schema.String,
timeout: Schema.String,
},
) {
override get message(): string {
return `Workspace search index for '${this.cwd}' did not finish scanning within ${this.timeout}`;
}
}
export class WorkspaceSearchIndexSearchFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexSearchFailed>()(
"WorkspaceSearchIndexSearchFailed",
{
cwd: Schema.String,
queryLength: Schema.Number,
pageSize: Schema.Number,
reason: Schema.String,
cause: Schema.optional(Schema.Defect()),
},
) {
override get message(): string {
return `Workspace search failed for '${this.cwd}'.`;
}
}
export class WorkspaceSearchIndexRefreshFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexRefreshFailed>()(
"WorkspaceSearchIndexRefreshFailed",
{
cwd: Schema.String,
reason: Schema.String,
cause: Schema.optional(Schema.Defect()),
},
) {
override get message(): string {
return `Failed to refresh the workspace search index for '${this.cwd}'.`;
}
}
export class WorkspaceSearchIndexDestroyFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexDestroyFailed>()(
"WorkspaceSearchIndexDestroyFailed",
{
cwd: Schema.String,
cause: Schema.Defect(),
},
) {
override get message(): string {
return `Failed to destroy the workspace search index for '${this.cwd}'.`;
}
}
export type WorkspaceSearchIndexError =
| WorkspaceSearchIndexCreateFailed
| WorkspaceSearchIndexScanTimedOut
| WorkspaceSearchIndexSearchFailed
| WorkspaceSearchIndexRefreshFailed;
export class WorkspaceSearchIndex extends Context.Service<
WorkspaceSearchIndex,
{
readonly list: () => Effect.Effect<ProjectListEntriesResult, WorkspaceSearchIndexSearchFailed>;
readonly search: (
query: string,
limit: number,
) => Effect.Effect<ProjectSearchEntriesResult, WorkspaceSearchIndexSearchFailed>;
readonly refresh: () => Effect.Effect<
void,
WorkspaceSearchIndexRefreshFailed | WorkspaceSearchIndexScanTimedOut
>;
}
>()("t3/workspace/WorkspaceSearchIndex") {}
function toPosixPath(input: string): string {
return input.replaceAll("\\", "/");
}
function trimDirectorySeparator(input: string): string {
return input.endsWith("/") ? input.slice(0, -1) : input;
}
function parentPathOf(input: string): string | undefined {
const separatorIndex = input.lastIndexOf("/");
return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex);
}
function toProjectEntry(item: MixedItem): ProjectEntry | null {
const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath));
if (!normalizedPath) {
return null;
}
return {
path: normalizedPath,
kind: item.type,
};
}
function mapMixedSearchResult(
result: MixedSearchResult,
limit: number,
): { readonly entries: ProjectEntry[]; readonly truncated: boolean } {
const entries: ProjectEntry[] = [];
for (const item of result.items) {
const entry = toProjectEntry(item);
if (entry) {
entries.push(entry);
}
if (entries.length >= limit) {
break;
}
}
const rootDirectoryCount = result.items.some(
(item) => item.type === "directory" && item.item.relativePath.length === 0,
)
? 1
: 0;
return {
entries,
truncated: result.totalMatched - rootDirectoryCount > limit,
};
}
function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEntry[] {
const entryByPath = new Map(entries.map((entry) => [entry.path, entry]));
for (const entry of entries) {
let parentPath = parentPathOf(entry.path);
while (parentPath) {
if (!entryByPath.has(parentPath)) {
entryByPath.set(parentPath, { path: parentPath, kind: "directory" });
}
parentPath = parentPathOf(parentPath);
}
}
return [...entryByPath.values()];
}
const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* (cwd: string) {
const result = yield* Effect.try({
try: () =>
FileFinder.create({
basePath: cwd,
disableMmapCache: true,
disableContentIndexing: true,
aiMode: false,
enableFsRootScanning: true,
enableHomeDirScanning: true,
}),
catch: (cause) =>
new WorkspaceSearchIndexCreateFailed({
cwd,
reason: "FileFinder.create threw unexpectedly.",
cause,
}),
});
if (result.ok) return result.value;
return yield* new WorkspaceSearchIndexCreateFailed({
cwd,
reason: result.error,
});
});
const waitForScan = <E>(cwd: string, finder: FileFinder, onFailure: (cause: unknown) => E) =>
Effect.try({
try: () => finder.isScanning(),
catch: onFailure,
}).pipe(
Effect.repeat({
while: (scanning) => scanning,
schedule: Schedule.spaced(WORKSPACE_INDEX_SCAN_POLL_INTERVAL),
}),
Effect.timeoutOrElse({
duration: WORKSPACE_INDEX_SCAN_TIMEOUT,
orElse: () =>
new WorkspaceSearchIndexScanTimedOut({ cwd, timeout: WORKSPACE_INDEX_SCAN_TIMEOUT }),
}),
Effect.withSpan("WorkspaceSearchIndex.waitForScan"),
);
export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (cwd: string) {
const finder = yield* Effect.acquireRelease(createFinder(cwd), (finder) =>
Effect.try({
try: () => finder.destroy(),
catch: (cause) => new WorkspaceSearchIndexDestroyFailed({ cwd, cause }),
}).pipe(Effect.orDie),
);
yield* waitForScan(
cwd,
finder,
(cause) =>
new WorkspaceSearchIndexCreateFailed({
cwd,
reason: "FileFinder.isScanning threw while creating the index.",
cause,
}),
);
const runMixedSearch = Effect.fn("WorkspaceSearchIndex.runMixedSearch")(function* (
query: string,
pageSize: number,
) {
const result = yield* Effect.try({
try: () => finder.mixedSearch(query, { pageSize }),
catch: (cause) =>
new WorkspaceSearchIndexSearchFailed({
cwd,
queryLength: query.length,
pageSize,
reason: "FileFinder.mixedSearch threw unexpectedly.",
cause,
}),
});
if (!result.ok) {
return yield* new WorkspaceSearchIndexSearchFailed({
cwd,
queryLength: query.length,
pageSize,
reason: result.error,
});
}
return result.value;
});
const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn(
"WorkspaceSearchIndex.refresh",
)(function* () {
const result = yield* Effect.try({
try: () => finder.scanFiles(),
catch: (cause) =>
new WorkspaceSearchIndexRefreshFailed({
cwd,
reason: "FileFinder.scanFiles threw unexpectedly.",
cause,
}),
});
if (!result.ok) {
return yield* new WorkspaceSearchIndexRefreshFailed({
cwd,
reason: result.error,
});
}
yield* waitForScan(
cwd,
finder,
(cause) =>
new WorkspaceSearchIndexRefreshFailed({
cwd,
reason: "FileFinder.isScanning threw while refreshing the index.",
cause,
}),
);
});
const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")(
function* () {
const result = yield* runMixedSearch("", WORKSPACE_INDEX_PAGE_SIZE);
const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES);
const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) =>
left.path.localeCompare(right.path),
);
const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES);
return {
entries,
truncated: mapped.truncated || entries.length < sortedEntries.length,
};
},
);
const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn(
"WorkspaceSearchIndex.search",
)(function* (query, limit) {
const result = yield* runMixedSearch(query, Math.max(1, limit + 1));
return mapMixedSearchResult(result, limit);
});
return WorkspaceSearchIndex.of({ list, refresh, search });
});
/**
* A layer factory is required because every index is scoped to a concrete
* workspace root. WorkspaceSearchIndexMap owns memoization and idle cleanup;
* using a default cwd here would mix resources from different workspaces.
*/
export const layer = (cwd: string) => Layer.effect(WorkspaceSearchIndex, make(cwd));
export class WorkspaceSearchIndexMap extends LayerMap.Service<WorkspaceSearchIndexMap>()(
"t3/workspace/WorkspaceSearchIndexMap",
{
lookup: layer,
idleTimeToLive: WORKSPACE_INDEX_IDLE_TTL,
},
) {}