import { FileFinder, type MixedItem, type MixedSearchResult } from "@ff-labs/fff-node";import * as Context from "effect/Context";import * as Effect from "effect/Effect";import * as Layer from "effect/Layer";import * as LayerMap from "effect/LayerMap";import * as Schedule from "effect/Schedule";import * as Schema from "effect/Schema"; import type { ProjectEntry, ProjectListEntriesResult, ProjectSearchEntriesResult,} from "@t3tools/contracts"; const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2;const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds";const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";const WORKSPACE_INDEX_SCAN_POLL_INTERVAL = "50 millis"; export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexCreateFailed>()( "WorkspaceSearchIndexCreateFailed", { cwd: Schema.String, reason: Schema.String, },) { override get message(): string { return `Failed to create the workspace search index for '${this.cwd}': ${this.reason}`; }} export class WorkspaceSearchIndexScanTimedOut extends Schema.TaggedErrorClass<WorkspaceSearchIndexScanTimedOut>()( "WorkspaceSearchIndexScanTimedOut", { cwd: Schema.String, timeout: Schema.String, },) { override get message(): string { return `Workspace search index for '${this.cwd}' did not finish scanning within ${this.timeout}`; }} export class WorkspaceSearchIndexSearchFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexSearchFailed>()( "WorkspaceSearchIndexSearchFailed", { cwd: Schema.String, reason: Schema.String, },) { override get message(): string { return `Workspace search failed for '${this.cwd}': ${this.reason}`; }} export class WorkspaceSearchIndexRefreshFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexRefreshFailed>()( "WorkspaceSearchIndexRefreshFailed", { cwd: Schema.String, reason: Schema.String, },) { override get message(): string { return `Failed to refresh the workspace search index for '${this.cwd}': ${this.reason}`; }} export type WorkspaceSearchIndexError = | WorkspaceSearchIndexCreateFailed | WorkspaceSearchIndexScanTimedOut | WorkspaceSearchIndexSearchFailed | WorkspaceSearchIndexRefreshFailed; export class WorkspaceSearchIndex extends Context.Service< WorkspaceSearchIndex, { readonly list: () => Effect.Effect<ProjectListEntriesResult, WorkspaceSearchIndexSearchFailed>; readonly search: ( query: string, limit: number, ) => Effect.Effect<ProjectSearchEntriesResult, WorkspaceSearchIndexSearchFailed>; readonly refresh: () => Effect.Effect< void, WorkspaceSearchIndexRefreshFailed | WorkspaceSearchIndexScanTimedOut >; }>()("t3/workspace/WorkspaceSearchIndex") {} function toPosixPath(input: string): string { return input.replaceAll("\\", "/");} function trimDirectorySeparator(input: string): string { return input.endsWith("/") ? input.slice(0, -1) : input;} function parentPathOf(input: string): string | undefined { const separatorIndex = input.lastIndexOf("/"); return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex);} function toProjectEntry(item: MixedItem): ProjectEntry | null { const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath)); if (!normalizedPath) { return null; } return { path: normalizedPath, kind: item.type, };} function mapMixedSearchResult( result: MixedSearchResult, limit: number,): { readonly entries: ProjectEntry[]; readonly truncated: boolean } { const entries: ProjectEntry[] = []; for (const item of result.items) { const entry = toProjectEntry(item); if (entry) { entries.push(entry); } if (entries.length >= limit) { break; } } const rootDirectoryCount = result.items.some( (item) => item.type === "directory" && item.item.relativePath.length === 0, ) ? 1 : 0; return { entries, truncated: result.totalMatched - rootDirectoryCount > limit, };} function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEntry[] { const entryByPath = new Map(entries.map((entry) => [entry.path, entry])); for (const entry of entries) { let parentPath = parentPathOf(entry.path); while (parentPath) { if (!entryByPath.has(parentPath)) { entryByPath.set(parentPath, { path: parentPath, kind: "directory" }); } parentPath = parentPathOf(parentPath); } } return [...entryByPath.values()];} const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* (cwd: string) { const result = FileFinder.create({ basePath: cwd, disableMmapCache: true, disableContentIndexing: true, aiMode: false, enableFsRootScanning: true, enableHomeDirScanning: true, }); if (result.ok) return result.value; return yield* new WorkspaceSearchIndexCreateFailed({ cwd, reason: result.error });}); const waitForScan = Effect.fn("WorkspaceSearchIndex.waitForScan")(function* ( cwd: string, finder: FileFinder,) { yield* Effect.sync(() => finder.isScanning()).pipe( Effect.repeat({ while: (scanning) => scanning, schedule: Schedule.spaced(WORKSPACE_INDEX_SCAN_POLL_INTERVAL), }), Effect.timeoutOrElse({ duration: WORKSPACE_INDEX_SCAN_TIMEOUT, orElse: () => new WorkspaceSearchIndexScanTimedOut({ cwd, timeout: WORKSPACE_INDEX_SCAN_TIMEOUT }), }), );}); const makeWorkspaceSearchIndex = (cwd: string) => Effect.acquireRelease(createFinder(cwd), (finder) => Effect.sync(() => finder.destroy())).pipe( Effect.tap((finder) => waitForScan(cwd, finder)), Effect.map((finder) => { const runMixedSearch = Effect.fn("WorkspaceSearchIndex.runMixedSearch")(function* ( query: string, pageSize: number, ) { const result = yield* Effect.sync(() => finder.mixedSearch(query, { pageSize })); if (!result.ok) { return yield* new WorkspaceSearchIndexSearchFailed({ cwd, reason: result.error }); } return result.value; }); const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn( "WorkspaceSearchIndex.refresh", )(function* () { const result = yield* Effect.sync(() => finder.scanFiles()); if (!result.ok) { return yield* new WorkspaceSearchIndexRefreshFailed({ cwd, reason: result.error }); } yield* waitForScan(cwd, finder); }); const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")( function* () { const result = yield* runMixedSearch("", WORKSPACE_INDEX_PAGE_SIZE); const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES); const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) => left.path.localeCompare(right.path), ); const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES); return { entries, truncated: mapped.truncated || entries.length < sortedEntries.length, }; }, ); const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn( "WorkspaceSearchIndex.search", )(function* (query, limit) { const result = yield* runMixedSearch(query, Math.max(1, limit + 1)); return mapMixedSearchResult(result, limit); }); return WorkspaceSearchIndex.of({ list, refresh, search }); }), ); const workspaceSearchIndexLayer = (cwd: string) => Layer.effect(WorkspaceSearchIndex, makeWorkspaceSearchIndex(cwd)); export class WorkspaceSearchIndexMap extends LayerMap.Service<WorkspaceSearchIndexMap>()( "t3/workspace/WorkspaceSearchIndexMap", { lookup: workspaceSearchIndexLayer, idleTimeToLive: WORKSPACE_INDEX_IDLE_TTL, },) {}
import { FileFinder, type MixedItem, type MixedSearchResult } from "@ff-labs/fff-node";
import * as Context from "effect/Context";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as LayerMap from "effect/LayerMap";
import * as Schedule from "effect/Schedule";
import * as Schema from "effect/Schema";
import type {
ProjectEntry,
ProjectListEntriesResult,
ProjectSearchEntriesResult,
} from "@t3tools/contracts";
const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;
const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2;
const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds";
const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
const WORKSPACE_INDEX_SCAN_POLL_INTERVAL = "50 millis";
export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexCreateFailed>()(
"WorkspaceSearchIndexCreateFailed",
{
cwd: Schema.String,
reason: Schema.String,
},
) {
override get message(): string {
return `Failed to create the workspace search index for '${this.cwd}': ${this.reason}`;
}
}
export class WorkspaceSearchIndexScanTimedOut extends Schema.TaggedErrorClass<WorkspaceSearchIndexScanTimedOut>()(
"WorkspaceSearchIndexScanTimedOut",
{
cwd: Schema.String,
timeout: Schema.String,
},
) {
override get message(): string {
return `Workspace search index for '${this.cwd}' did not finish scanning within ${this.timeout}`;
}
}
export class WorkspaceSearchIndexSearchFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexSearchFailed>()(
"WorkspaceSearchIndexSearchFailed",
{
cwd: Schema.String,
reason: Schema.String,
},
) {
override get message(): string {
return `Workspace search failed for '${this.cwd}': ${this.reason}`;
}
}
export class WorkspaceSearchIndexRefreshFailed extends Schema.TaggedErrorClass<WorkspaceSearchIndexRefreshFailed>()(
"WorkspaceSearchIndexRefreshFailed",
{
cwd: Schema.String,
reason: Schema.String,
},
) {
override get message(): string {
return `Failed to refresh the workspace search index for '${this.cwd}': ${this.reason}`;
}
}
export type WorkspaceSearchIndexError =
| WorkspaceSearchIndexCreateFailed
| WorkspaceSearchIndexScanTimedOut
| WorkspaceSearchIndexSearchFailed
| WorkspaceSearchIndexRefreshFailed;
export class WorkspaceSearchIndex extends Context.Service<
WorkspaceSearchIndex,
{
readonly list: () => Effect.Effect<ProjectListEntriesResult, WorkspaceSearchIndexSearchFailed>;
readonly search: (
query: string,
limit: number,
) => Effect.Effect<ProjectSearchEntriesResult, WorkspaceSearchIndexSearchFailed>;
readonly refresh: () => Effect.Effect<
void,
WorkspaceSearchIndexRefreshFailed | WorkspaceSearchIndexScanTimedOut
>;
}
>()("t3/workspace/WorkspaceSearchIndex") {}
function toPosixPath(input: string): string {
return input.replaceAll("\\", "/");
}
function trimDirectorySeparator(input: string): string {
return input.endsWith("/") ? input.slice(0, -1) : input;
}
function parentPathOf(input: string): string | undefined {
const separatorIndex = input.lastIndexOf("/");
return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex);
}
function toProjectEntry(item: MixedItem): ProjectEntry | null {
const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath));
if (!normalizedPath) {
return null;
}
return {
path: normalizedPath,
kind: item.type,
};
}
function mapMixedSearchResult(
result: MixedSearchResult,
limit: number,
): { readonly entries: ProjectEntry[]; readonly truncated: boolean } {
const entries: ProjectEntry[] = [];
for (const item of result.items) {
const entry = toProjectEntry(item);
if (entry) {
entries.push(entry);
}
if (entries.length >= limit) {
break;
}
}
const rootDirectoryCount = result.items.some(
(item) => item.type === "directory" && item.item.relativePath.length === 0,
)
? 1
: 0;
return {
entries,
truncated: result.totalMatched - rootDirectoryCount > limit,
};
}
function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEntry[] {
const entryByPath = new Map(entries.map((entry) => [entry.path, entry]));
for (const entry of entries) {
let parentPath = parentPathOf(entry.path);
while (parentPath) {
if (!entryByPath.has(parentPath)) {
entryByPath.set(parentPath, { path: parentPath, kind: "directory" });
}
parentPath = parentPathOf(parentPath);
}
}
return [...entryByPath.values()];
}
const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* (cwd: string) {
const result = FileFinder.create({
basePath: cwd,
disableMmapCache: true,
disableContentIndexing: true,
aiMode: false,
enableFsRootScanning: true,
enableHomeDirScanning: true,
});
if (result.ok) return result.value;
return yield* new WorkspaceSearchIndexCreateFailed({ cwd, reason: result.error });
});
const waitForScan = Effect.fn("WorkspaceSearchIndex.waitForScan")(function* (
cwd: string,
finder: FileFinder,
) {
yield* Effect.sync(() => finder.isScanning()).pipe(
Effect.repeat({
while: (scanning) => scanning,
schedule: Schedule.spaced(WORKSPACE_INDEX_SCAN_POLL_INTERVAL),
}),
Effect.timeoutOrElse({
duration: WORKSPACE_INDEX_SCAN_TIMEOUT,
orElse: () =>
new WorkspaceSearchIndexScanTimedOut({ cwd, timeout: WORKSPACE_INDEX_SCAN_TIMEOUT }),
}),
);
});
const makeWorkspaceSearchIndex = (cwd: string) =>
Effect.acquireRelease(createFinder(cwd), (finder) => Effect.sync(() => finder.destroy())).pipe(
Effect.tap((finder) => waitForScan(cwd, finder)),
Effect.map((finder) => {
const runMixedSearch = Effect.fn("WorkspaceSearchIndex.runMixedSearch")(function* (
query: string,
pageSize: number,
) {
const result = yield* Effect.sync(() => finder.mixedSearch(query, { pageSize }));
if (!result.ok) {
return yield* new WorkspaceSearchIndexSearchFailed({ cwd, reason: result.error });
}
return result.value;
});
const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn(
"WorkspaceSearchIndex.refresh",
)(function* () {
const result = yield* Effect.sync(() => finder.scanFiles());
if (!result.ok) {
return yield* new WorkspaceSearchIndexRefreshFailed({ cwd, reason: result.error });
}
yield* waitForScan(cwd, finder);
});
const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")(
function* () {
const result = yield* runMixedSearch("", WORKSPACE_INDEX_PAGE_SIZE);
const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES);
const sortedEntries = withDirectoryAncestors(mapped.entries).toSorted((left, right) =>
left.path.localeCompare(right.path),
);
const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES);
return {
entries,
truncated: mapped.truncated || entries.length < sortedEntries.length,
};
},
);
const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn(
"WorkspaceSearchIndex.search",
)(function* (query, limit) {
const result = yield* runMixedSearch(query, Math.max(1, limit + 1));
return mapMixedSearchResult(result, limit);
});
return WorkspaceSearchIndex.of({ list, refresh, search });
}),
);
const workspaceSearchIndexLayer = (cwd: string) =>
Layer.effect(WorkspaceSearchIndex, makeWorkspaceSearchIndex(cwd));
export class WorkspaceSearchIndexMap extends LayerMap.Service<WorkspaceSearchIndexMap>()(
"t3/workspace/WorkspaceSearchIndexMap",
{
lookup: workspaceSearchIndexLayer,
idleTimeToLive: WORKSPACE_INDEX_IDLE_TTL,
},
) {}