diff --git a/packages/mcp/src/handlers.ts b/packages/mcp/src/handlers.ts index 1530d0c3..ec90b73f 100644 --- a/packages/mcp/src/handlers.ts +++ b/packages/mcp/src/handlers.ts @@ -19,14 +19,12 @@ export class ToolHandlers { } /** - * Sync indexed codebases from Zilliz Cloud collections - * This method fetches all collections from the vector database, - * gets the first document from each collection to extract codebasePath from metadata, - * and updates the snapshot with discovered codebases. - * - * Logic: Compare mcp-codebase-snapshot.json with zilliz cloud collections - * - If local snapshot has extra directories (not in cloud), remove them - * - If local snapshot is missing directories (exist in cloud), ignore them + * Best-effort cloud sync for diagnostics. + * + * IMPORTANT SAFETY RULE: + * Never remove local snapshot entries based only on cloud list/query results. + * Different CLI sessions may run with different credentials/clusters, and + * transient cloud visibility issues can cause false negatives. */ private async syncIndexedCodebasesFromCloud(): Promise { try { @@ -41,18 +39,7 @@ export class ToolHandlers { console.log(`[SYNC-CLOUD] ๐Ÿ“‹ Found ${collections.length} collections in Zilliz Cloud`); if (collections.length === 0) { - console.log(`[SYNC-CLOUD] โœ… No collections found in cloud`); - // If no collections in cloud, remove all local codebases - const localCodebases = this.snapshotManager.getIndexedCodebases(); - if (localCodebases.length > 0) { - console.log(`[SYNC-CLOUD] ๐Ÿงน Removing ${localCodebases.length} local codebases as cloud has no collections`); - for (const codebasePath of localCodebases) { - this.snapshotManager.removeIndexedCodebase(codebasePath); - console.log(`[SYNC-CLOUD] โž– Removed local codebase: ${codebasePath}`); - } - this.snapshotManager.saveCodebaseSnapshot(); - console.log(`[SYNC-CLOUD] ๐Ÿ’พ Updated snapshot to match empty cloud state`); - } + console.warn(`[SYNC-CLOUD] โš ๏ธ Cloud returned zero collections. Skipping local snapshot cleanup to avoid false negatives.`); return; } @@ -110,30 +97,19 @@ export class ToolHandlers { console.log(`[SYNC-CLOUD] ๐Ÿ“Š Found ${cloudCodebases.size} valid codebases in cloud`); // Get current local codebases - const localCodebases = new Set(this.snapshotManager.getIndexedCodebases()); - console.log(`[SYNC-CLOUD] ๐Ÿ“Š Found ${localCodebases.size} local codebases in snapshot`); - - let hasChanges = false; + const localCodebases = this.snapshotManager.getIndexedCodebases(); + console.log(`[SYNC-CLOUD] ๐Ÿ“Š Found ${localCodebases.length} local codebases in snapshot`); - // Remove local codebases that don't exist in cloud - for (const localCodebase of localCodebases) { - if (!cloudCodebases.has(localCodebase)) { - this.snapshotManager.removeIndexedCodebase(localCodebase); - hasChanges = true; - console.log(`[SYNC-CLOUD] โž– Removed local codebase (not in cloud): ${localCodebase}`); - } - } - - // Note: We don't add cloud codebases that are missing locally (as per user requirement) - console.log(`[SYNC-CLOUD] โ„น๏ธ Skipping addition of cloud codebases not present locally (per sync policy)`); - - if (hasChanges) { - this.snapshotManager.saveCodebaseSnapshot(); - console.log(`[SYNC-CLOUD] ๐Ÿ’พ Updated snapshot to match cloud state`); - } else { - console.log(`[SYNC-CLOUD] โœ… Local snapshot already matches cloud state`); + const missingInCloud = localCodebases.filter((localCodebase) => !cloudCodebases.has(localCodebase)); + if (missingInCloud.length > 0) { + console.warn( + `[SYNC-CLOUD] โš ๏ธ ${missingInCloud.length} local codebase(s) were not found in cloud metadata. ` + + `Keeping local snapshot unchanged for safety.` + ); } + // Note: intentionally no snapshot mutation here. + console.log(`[SYNC-CLOUD] โ„น๏ธ Cloud sync is non-destructive; local snapshot was not modified.`); console.log(`[SYNC-CLOUD] โœ… Cloud sync completed successfully`); } catch (error: any) { console.error(`[SYNC-CLOUD] โŒ Error syncing codebases from cloud:`, error.message || error); @@ -188,8 +164,10 @@ export class ToolHandlers { }; } - // Check if already indexing - if (this.snapshotManager.getIndexingCodebases().includes(absolutePath)) { + // Check if this process is already indexing the codebase. + // We intentionally rely on in-memory status here so stale on-disk "indexing" + // entries from previous MCP sessions don't block force reindex. + if (this.snapshotManager.getCodebaseStatus(absolutePath) === 'indexing') { return { content: [{ type: "text", @@ -199,13 +177,29 @@ export class ToolHandlers { }; } - //Check if the snapshot and cloud index are in sync - if (this.snapshotManager.getIndexedCodebases().includes(absolutePath) !== await this.context.hasIndex(absolutePath)) { + const snapshotHasIndex = this.snapshotManager.getIndexedCodebases().includes(absolutePath); + const cloudHasIndex = await this.context.hasIndex(absolutePath); + + // Reconcile local snapshot with cloud truth for this specific codebase + if (snapshotHasIndex !== cloudHasIndex) { console.warn(`[INDEX-VALIDATION] โŒ Snapshot and cloud index mismatch: ${absolutePath}`); + if (cloudHasIndex && !snapshotHasIndex) { + this.snapshotManager.setCodebaseIndexed(absolutePath, { + indexedFiles: 0, + totalChunks: 0, + status: 'completed' + }); + await this.snapshotManager.saveCodebaseSnapshot('index-reconcile-cloud-present'); + console.log(`[INDEX-VALIDATION] ๐Ÿ› ๏ธ Recovered missing snapshot entry from cloud index: ${absolutePath}`); + } else if (!cloudHasIndex && snapshotHasIndex) { + this.snapshotManager.removeCodebaseCompletely(absolutePath); + await this.snapshotManager.saveCodebaseSnapshot('index-reconcile-cloud-missing'); + console.log(`[INDEX-VALIDATION] ๐Ÿงน Removed stale snapshot entry without cloud index: ${absolutePath}`); + } } - // Check if already indexed (unless force is true) - if (!forceReindex && this.snapshotManager.getIndexedCodebases().includes(absolutePath)) { + // Check if already indexed in cloud (unless force is true) + if (!forceReindex && cloudHasIndex) { return { content: [{ type: "text", @@ -219,9 +213,9 @@ export class ToolHandlers { if (forceReindex) { if (this.snapshotManager.getIndexedCodebases().includes(absolutePath)) { console.log(`[FORCE-REINDEX] ๐Ÿ”„ Removing '${absolutePath}' from indexed list for re-indexing`); - this.snapshotManager.removeIndexedCodebase(absolutePath); + this.snapshotManager.removeCodebaseCompletely(absolutePath); } - if (await this.context.hasIndex(absolutePath)) { + if (cloudHasIndex) { console.log(`[FORCE-REINDEX] ๐Ÿ”„ Clearing index for '${absolutePath}'`); await this.context.clearIndex(absolutePath); } @@ -279,7 +273,7 @@ export class ToolHandlers { // Set to indexing status and save snapshot immediately this.snapshotManager.setCodebaseIndexing(absolutePath, 0); - this.snapshotManager.saveCodebaseSnapshot(); + await this.snapshotManager.saveCodebaseSnapshot('index-started'); // Track the codebase path for syncing trackCodebasePath(absolutePath); @@ -323,7 +317,7 @@ export class ToolHandlers { private async startBackgroundIndexing(codebasePath: string, forceReindex: boolean, splitterType: string) { const absolutePath = codebasePath; - let lastSaveTime = 0; // Track last save timestamp + let lastPersistedProgress = -1; try { console.log(`[BACKGROUND-INDEX] Starting background indexing for: ${absolutePath}`); @@ -369,12 +363,16 @@ export class ToolHandlers { // Update progress in snapshot manager using new method this.snapshotManager.setCodebaseIndexing(absolutePath, progress.percentage); - // Save snapshot periodically (every 2 seconds to avoid too frequent saves) - const currentTime = Date.now(); - if (currentTime - lastSaveTime >= 2000) { // 2 seconds = 2000ms - this.snapshotManager.saveCodebaseSnapshot(); - lastSaveTime = currentTime; - console.log(`[BACKGROUND-INDEX] ๐Ÿ’พ Saved progress snapshot at ${progress.percentage.toFixed(1)}%`); + // Coalesce disk writes: persist only meaningful progress jumps. + const shouldPersistProgress = + lastPersistedProgress < 0 || + progress.percentage >= 100 || + Math.abs(progress.percentage - lastPersistedProgress) >= 2; + + if (shouldPersistProgress) { + this.snapshotManager.scheduleSaveCodebaseSnapshot('index-progress'); + lastPersistedProgress = progress.percentage; + console.log(`[BACKGROUND-INDEX] ๐Ÿ’พ Scheduled progress snapshot at ${progress.percentage.toFixed(1)}%`); } console.log(`[BACKGROUND-INDEX] Progress: ${progress.phase} - ${progress.percentage}% (${progress.current}/${progress.total})`); @@ -386,7 +384,7 @@ export class ToolHandlers { this.indexingStats = { indexedFiles: stats.indexedFiles, totalChunks: stats.totalChunks }; // Save snapshot after updating codebase lists - this.snapshotManager.saveCodebaseSnapshot(); + await this.snapshotManager.saveCodebaseSnapshot('index-completed'); let message = `Background indexing completed for '${absolutePath}' using ${splitterType.toUpperCase()} splitter.\nIndexed ${stats.indexedFiles} files, ${stats.totalChunks} chunks.`; if (stats.status === 'limit_reached') { @@ -404,7 +402,7 @@ export class ToolHandlers { // Set codebase to failed status with error information const errorMessage = error.message || String(error); this.snapshotManager.setCodebaseIndexFailed(absolutePath, errorMessage, lastProgress); - this.snapshotManager.saveCodebaseSnapshot(); + await this.snapshotManager.saveCodebaseSnapshot('index-failed'); // Log error but don't crash MCP service - indexing errors are handled gracefully console.error(`[BACKGROUND-INDEX] Indexing failed for ${absolutePath}: ${errorMessage}`); @@ -447,11 +445,23 @@ export class ToolHandlers { trackCodebasePath(absolutePath); - // Check if this codebase is indexed or being indexed - const isIndexed = this.snapshotManager.getIndexedCodebases().includes(absolutePath); + // Check status with cloud as source of truth and snapshot as progress source + const isIndexedInSnapshot = this.snapshotManager.getIndexedCodebases().includes(absolutePath); const isIndexing = this.snapshotManager.getIndexingCodebases().includes(absolutePath); + const hasCloudIndex = await this.context.hasIndex(absolutePath); + + // Self-heal snapshot if index exists in cloud but local snapshot is missing + if (hasCloudIndex && !isIndexedInSnapshot && !isIndexing) { + this.snapshotManager.setCodebaseIndexed(absolutePath, { + indexedFiles: 0, + totalChunks: 0, + status: 'completed' + }); + await this.snapshotManager.saveCodebaseSnapshot('search-reconcile-cloud-present'); + console.log(`[SEARCH] ๐Ÿ› ๏ธ Restored missing snapshot entry from cloud index for: ${absolutePath}`); + } - if (!isIndexed && !isIndexing) { + if (!hasCloudIndex && !isIndexing) { return { content: [{ type: "text", @@ -469,7 +479,7 @@ export class ToolHandlers { console.log(`[SEARCH] Searching in codebase: ${absolutePath}`); console.log(`[SEARCH] Query: "${query}"`); - console.log(`[SEARCH] Indexing status: ${isIndexing ? 'In Progress' : 'Completed'}`); + console.log(`[SEARCH] Indexing status: ${isIndexing ? 'In Progress' : (hasCloudIndex ? 'Completed' : 'No collection yet')}`); // Log embedding provider information before search const embeddingProvider = this.context.getEmbedding(); @@ -571,15 +581,6 @@ export class ToolHandlers { public async handleClearIndex(args: any) { const { path: codebasePath } = args; - if (this.snapshotManager.getIndexedCodebases().length === 0 && this.snapshotManager.getIndexingCodebases().length === 0) { - return { - content: [{ - type: "text", - text: "No codebases are currently indexed or being indexed." - }] - }; - } - try { // Force absolute path resolution - warn if relative path provided const absolutePath = ensureAbsolutePath(codebasePath); @@ -610,8 +611,9 @@ export class ToolHandlers { // Check if this codebase is indexed or being indexed const isIndexed = this.snapshotManager.getIndexedCodebases().includes(absolutePath); const isIndexing = this.snapshotManager.getIndexingCodebases().includes(absolutePath); + const hasCloudIndex = await this.context.hasIndex(absolutePath); - if (!isIndexed && !isIndexing) { + if (!isIndexed && !isIndexing && !hasCloudIndex) { return { content: [{ type: "text", @@ -623,19 +625,23 @@ export class ToolHandlers { console.log(`[CLEAR] Clearing codebase: ${absolutePath}`); - try { - await this.context.clearIndex(absolutePath); - console.log(`[CLEAR] Successfully cleared index for: ${absolutePath}`); - } catch (error: any) { - const errorMsg = `Failed to clear ${absolutePath}: ${error.message}`; - console.error(`[CLEAR] ${errorMsg}`); - return { - content: [{ - type: "text", - text: errorMsg - }], - isError: true - }; + if (hasCloudIndex) { + try { + await this.context.clearIndex(absolutePath); + console.log(`[CLEAR] Successfully cleared index for: ${absolutePath}`); + } catch (error: any) { + const errorMsg = `Failed to clear ${absolutePath}: ${error.message}`; + console.error(`[CLEAR] ${errorMsg}`); + return { + content: [{ + type: "text", + text: errorMsg + }], + isError: true + }; + } + } else { + console.log(`[CLEAR] โ„น๏ธ No cloud collection found for ${absolutePath}, cleaning snapshot only`); } // Completely remove the cleared codebase from snapshot @@ -645,7 +651,7 @@ export class ToolHandlers { this.indexingStats = null; // Save snapshot after clearing index - this.snapshotManager.saveCodebaseSnapshot(); + await this.snapshotManager.saveCodebaseSnapshot('clear-index'); let resultText = `Successfully cleared codebase '${absolutePath}'`; @@ -718,9 +724,34 @@ export class ToolHandlers { }; } - // Check indexing status using new status system - const status = this.snapshotManager.getCodebaseStatus(absolutePath); - const info = this.snapshotManager.getCodebaseInfo(absolutePath); + // Check indexing status using snapshot plus cloud truth + let status = this.snapshotManager.getCodebaseStatus(absolutePath); + let info = this.snapshotManager.getCodebaseInfo(absolutePath); + let recoveredFromCloud = false; + const hasCloudIndex = await this.context.hasIndex(absolutePath); + + // Self-heal snapshot if cloud has index but local status is missing + if (status === 'not_found' && hasCloudIndex) { + this.snapshotManager.setCodebaseIndexed(absolutePath, { + indexedFiles: 0, + totalChunks: 0, + status: 'completed' + }); + await this.snapshotManager.saveCodebaseSnapshot('status-reconcile-cloud-present'); + status = 'indexed'; + info = this.snapshotManager.getCodebaseInfo(absolutePath); + recoveredFromCloud = true; + console.log(`[STATUS] ๐Ÿ› ๏ธ Restored missing snapshot entry from cloud index for: ${absolutePath}`); + } + + // Cleanup stale snapshot entries if cloud index no longer exists + if (status === 'indexed' && !hasCloudIndex) { + this.snapshotManager.removeCodebaseCompletely(absolutePath); + await this.snapshotManager.saveCodebaseSnapshot('status-reconcile-cloud-missing'); + status = 'not_found'; + info = undefined; + console.log(`[STATUS] ๐Ÿงน Removed stale indexed snapshot entry without cloud index for: ${absolutePath}`); + } let statusMessage = ''; @@ -735,6 +766,9 @@ export class ToolHandlers { } else { statusMessage = `โœ… Codebase '${absolutePath}' is fully indexed and ready for search.`; } + if (recoveredFromCloud) { + statusMessage += `\nโ„น๏ธ Index was detected directly in vector database and local snapshot state was restored.`; + } break; case 'indexing': @@ -797,4 +831,4 @@ export class ToolHandlers { }; } } -} \ No newline at end of file +} diff --git a/packages/mcp/src/snapshot.ts b/packages/mcp/src/snapshot.ts index 81982b7b..d690d07e 100644 --- a/packages/mcp/src/snapshot.ts +++ b/packages/mcp/src/snapshot.ts @@ -1,6 +1,7 @@ import * as fs from "fs"; import * as path from "path"; import * as os from "os"; +import * as crypto from "crypto"; import { CodebaseSnapshot, CodebaseSnapshotV1, @@ -11,16 +12,466 @@ import { CodebaseInfoIndexFailed } from "./config.js"; +type SnapshotScope = 'workspace' | 'global'; + +interface SnapshotManagerOptions { + workspacePath?: string; + scope?: SnapshotScope; + saveDebounceMs?: number; +} + export class SnapshotManager { private snapshotFilePath: string; + private lockFilePath: string; + private legacySnapshotFilePath: string; + private scope: SnapshotScope; + private workspacePath: string; private indexedCodebases: string[] = []; private indexingCodebases: Map = new Map(); // Map of codebase path to progress percentage private codebaseFileCount: Map = new Map(); // Map of codebase path to indexed file count private codebaseInfoMap: Map = new Map(); // Map of codebase path to complete info + private pendingDeletes: Map = new Map(); // Map of codebase path to delete timestamp + private readonly lockAcquireTimeoutMs: number; + private readonly lockRetryIntervalMs: number; + private readonly lockRetryJitterMs: number; + private readonly lockStaleMs: number; + private readonly saveDebounceMs: number; + private pendingSaveTimer: ReturnType | null = null; + private pendingSaveReason: string | null = null; + private saveQueue: Promise = Promise.resolve(); + private lockWaitMsTotal = 0; + private lockRetryCountTotal = 0; + private lockTimeoutCount = 0; + + constructor(options: SnapshotManagerOptions = {}) { + this.workspacePath = path.resolve(options.workspacePath || process.cwd()); + this.scope = options.scope || this.resolveSnapshotScope(); + this.lockAcquireTimeoutMs = this.parsePositiveNumber(process.env.MCP_SNAPSHOT_LOCK_TIMEOUT_MS, 15000); + this.lockRetryIntervalMs = this.parsePositiveNumber(process.env.MCP_SNAPSHOT_LOCK_RETRY_MS, 50); + this.lockRetryJitterMs = this.parsePositiveNumber(process.env.MCP_SNAPSHOT_LOCK_JITTER_MS, 40); + this.lockStaleMs = this.parsePositiveNumber(process.env.MCP_SNAPSHOT_LOCK_STALE_MS, 120000); + this.saveDebounceMs = this.parsePositiveNumber( + process.env.MCP_SNAPSHOT_SAVE_DEBOUNCE_MS, + options.saveDebounceMs ?? 2000 + ); + this.legacySnapshotFilePath = path.join(os.homedir(), '.context', 'mcp-codebase-snapshot.json'); + this.snapshotFilePath = this.resolveSnapshotPath(); + this.lockFilePath = `${this.snapshotFilePath}.lock`; + + console.log( + `[SNAPSHOT-DEBUG] Snapshot scope='${this.scope}', workspace='${this.workspacePath}', file='${this.snapshotFilePath}'` + ); + } + + private parsePositiveNumber(rawValue: string | undefined, fallback: number): number { + if (!rawValue) { + return fallback; + } + + const value = Number(rawValue); + if (Number.isFinite(value) && value > 0) { + return value; + } + + return fallback; + } + + private resolveSnapshotScope(): SnapshotScope { + const rawScope = (process.env.MCP_SNAPSHOT_SCOPE || 'workspace').toLowerCase(); + return rawScope === 'global' ? 'global' : 'workspace'; + } + + private resolveSnapshotPath(): string { + if (this.scope === 'global') { + return this.legacySnapshotFilePath; + } + + const workspaceHash = crypto + .createHash('sha256') + .update(this.workspacePath) + .digest('hex') + .slice(0, 16); + + return path.join(os.homedir(), '.context', 'mcp', workspaceHash, 'mcp-codebase-snapshot.json'); + } + + private isPathWithinWorkspace(candidatePath: string): boolean { + const absoluteCandidate = path.resolve(candidatePath); + return absoluteCandidate === this.workspacePath || absoluteCandidate.startsWith(`${this.workspacePath}${path.sep}`); + } + + private filterSnapshotForWorkspace(snapshot: CodebaseSnapshotV2): CodebaseSnapshotV2 { + const filteredCodebases: Record = {}; + + for (const [codebasePath, info] of Object.entries(snapshot.codebases)) { + if (this.isPathWithinWorkspace(codebasePath)) { + filteredCodebases[codebasePath] = info; + } + } + + return { + formatVersion: 'v2', + codebases: filteredCodebases, + lastUpdated: new Date().toISOString() + }; + } + + private migrateLegacySnapshotIfNeeded(): void { + if (this.scope !== 'workspace') { + return; + } + + if (fs.existsSync(this.snapshotFilePath)) { + return; + } + + const legacySnapshot = this.readSnapshotFileUnsafe(this.legacySnapshotFilePath, false); + if (!legacySnapshot) { + return; + } + + const filteredSnapshot = this.filterSnapshotForWorkspace(legacySnapshot); + const migratedCount = Object.keys(filteredSnapshot.codebases).length; + if (migratedCount === 0) { + return; + } + + this.writeSnapshotToDiskUnsafe(filteredSnapshot, this.snapshotFilePath); + console.log( + `[SNAPSHOT-DEBUG] Migrated ${migratedCount} codebase(s) from legacy snapshot to workspace-scoped snapshot` + ); + } + + private async sleep(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); + } + + private async cleanupStaleLockFileIfNeeded(): Promise { + try { + const stat = await fs.promises.stat(this.lockFilePath); + if ((Date.now() - stat.mtimeMs) > this.lockStaleMs) { + await fs.promises.unlink(this.lockFilePath); + console.warn(`[SNAPSHOT-DEBUG] Removed stale snapshot lock: ${this.lockFilePath}`); + } + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.warn('[SNAPSHOT-DEBUG] Failed to inspect snapshot lock file:', error); + } + } + } + + private async withSnapshotLock(callback: () => Promise): Promise { + const lockDir = path.dirname(this.lockFilePath); + await fs.promises.mkdir(lockDir, { recursive: true }); + + const start = Date.now(); + let retryCount = 0; + + while (true) { + let lockHandle: fs.promises.FileHandle | null = null; + + try { + lockHandle = await fs.promises.open(this.lockFilePath, 'wx'); + await lockHandle.writeFile(`${process.pid}:${Date.now()}`); + + const lockWaitMs = Date.now() - start; + this.lockWaitMsTotal += lockWaitMs; + this.lockRetryCountTotal += retryCount; + + if (retryCount > 0 || lockWaitMs >= this.lockRetryIntervalMs) { + console.log( + `[SNAPSHOT-LOCK] wait_ms=${lockWaitMs} retries=${retryCount} total_wait_ms=${this.lockWaitMsTotal} total_retries=${this.lockRetryCountTotal}` + ); + } + + try { + return await callback(); + } finally { + if (lockHandle !== null) { + try { + await lockHandle.close(); + } catch { + // Ignore close errors + } + } + try { + await fs.promises.unlink(this.lockFilePath); + } catch (error: any) { + if (error.code !== 'ENOENT') { + console.warn('[SNAPSHOT-DEBUG] Failed to remove snapshot lock file:', error); + } + } + } + } catch (error: any) { + if (lockHandle !== null) { + try { + await lockHandle.close(); + } catch { + // Ignore close errors + } + } + + if (error.code !== 'EEXIST') { + throw error; + } + + retryCount += 1; + await this.cleanupStaleLockFileIfNeeded(); + + if ((Date.now() - start) >= this.lockAcquireTimeoutMs) { + this.lockTimeoutCount += 1; + const waitedMs = Date.now() - start; + throw new Error( + `Timeout acquiring snapshot lock: ${this.lockFilePath} (waited ${waitedMs}ms, retries ${retryCount}, total_timeouts ${this.lockTimeoutCount})` + ); + } + + const jitter = this.lockRetryJitterMs > 0 + ? Math.floor(Math.random() * this.lockRetryJitterMs) + : 0; + + await this.sleep(this.lockRetryIntervalMs + jitter); + } + } + } + + private buildSnapshotFromMemory(): CodebaseSnapshotV2 { + const codebases: Record = {}; + + for (const [codebasePath, info] of this.codebaseInfoMap) { + codebases[codebasePath] = info; + } + + return { + formatVersion: 'v2', + codebases, + lastUpdated: new Date().toISOString() + }; + } + + private applySnapshotToMemory(snapshot: CodebaseSnapshotV2): void { + const indexedCodebases: string[] = []; + const indexingCodebases = new Map(); + const codebaseFileCount = new Map(); + const codebaseInfoMap = new Map(); + + for (const [codebasePath, info] of Object.entries(snapshot.codebases)) { + codebaseInfoMap.set(codebasePath, info); + + if (info.status === 'indexed') { + indexedCodebases.push(codebasePath); + codebaseFileCount.set(codebasePath, info.indexedFiles || 0); + } else if (info.status === 'indexing') { + indexingCodebases.set(codebasePath, info.indexingPercentage || 0); + } + } + + this.indexedCodebases = indexedCodebases; + this.indexingCodebases = indexingCodebases; + this.codebaseFileCount = codebaseFileCount; + this.codebaseInfoMap = codebaseInfoMap; + } + + private convertV1ToV2(snapshot: CodebaseSnapshotV1): CodebaseSnapshotV2 { + const codebases: Record = {}; + const now = new Date().toISOString(); + const snapshotTime = snapshot.lastUpdated || now; + + for (const codebasePath of snapshot.indexedCodebases || []) { + codebases[codebasePath] = { + status: 'indexed', + indexedFiles: 0, + totalChunks: 0, + indexStatus: 'completed', + lastUpdated: snapshotTime + }; + } + + const indexingCodebases = snapshot.indexingCodebases; + if (Array.isArray(indexingCodebases)) { + for (const codebasePath of indexingCodebases) { + codebases[codebasePath] = { + status: 'indexing', + indexingPercentage: 0, + lastUpdated: snapshotTime + }; + } + } else if (indexingCodebases && typeof indexingCodebases === 'object') { + for (const [codebasePath, progress] of Object.entries(indexingCodebases)) { + codebases[codebasePath] = { + status: 'indexing', + indexingPercentage: typeof progress === 'number' ? progress : 0, + lastUpdated: snapshotTime + }; + } + } + + return { + formatVersion: 'v2', + codebases, + lastUpdated: snapshotTime + }; + } + + private readSnapshotFileUnsafe(snapshotPath: string, rotateCorruptFile: boolean): CodebaseSnapshotV2 | null { + if (!fs.existsSync(snapshotPath)) { + return null; + } + + try { + const snapshotData = fs.readFileSync(snapshotPath, 'utf8'); + const snapshot: CodebaseSnapshot = JSON.parse(snapshotData); - constructor() { - // Initialize snapshot file path - this.snapshotFilePath = path.join(os.homedir(), '.context', 'mcp-codebase-snapshot.json'); + if (this.isV2Format(snapshot)) { + return snapshot; + } + + return this.convertV1ToV2(snapshot); + } catch (error: any) { + console.warn('[SNAPSHOT-DEBUG] Failed to parse snapshot from disk:', error); + if (!rotateCorruptFile) { + return null; + } + + try { + const corruptedPath = `${snapshotPath}.corrupt.${Date.now()}`; + fs.renameSync(snapshotPath, corruptedPath); + console.warn(`[SNAPSHOT-DEBUG] Corrupted snapshot moved to: ${corruptedPath}`); + } catch (rotateError: any) { + if (rotateError.code !== 'ENOENT') { + console.warn('[SNAPSHOT-DEBUG] Failed to rotate corrupted snapshot file:', rotateError); + } + } + return null; + } + } + + private readSnapshotFromDiskUnsafe(): CodebaseSnapshotV2 | null { + return this.readSnapshotFileUnsafe(this.snapshotFilePath, true); + } + + private writeSnapshotToDiskUnsafe(snapshot: CodebaseSnapshotV2, snapshotPath: string = this.snapshotFilePath): void { + const snapshotDir = path.dirname(snapshotPath); + if (!fs.existsSync(snapshotDir)) { + fs.mkdirSync(snapshotDir, { recursive: true }); + console.log('[SNAPSHOT-DEBUG] Created snapshot directory:', snapshotDir); + } + + const tempPath = `${snapshotPath}.${process.pid}.${Date.now()}.tmp`; + try { + fs.writeFileSync(tempPath, JSON.stringify(snapshot, null, 2)); + fs.renameSync(tempPath, snapshotPath); + } catch (error) { + try { + if (fs.existsSync(tempPath)) { + fs.unlinkSync(tempPath); + } + } catch { + // Ignore temp file cleanup errors + } + throw error; + } + } + + private toTimestamp(value: string | undefined): number { + if (!value) { + return 0; + } + + const parsed = Date.parse(value); + return Number.isNaN(parsed) ? 0 : parsed; + } + + private mergeSnapshots(existingSnapshot: CodebaseSnapshotV2 | null, localSnapshot: CodebaseSnapshotV2): CodebaseSnapshotV2 { + const mergedCodebases: Record = {}; + + if (existingSnapshot) { + for (const [codebasePath, info] of Object.entries(existingSnapshot.codebases)) { + mergedCodebases[codebasePath] = info; + } + } + + for (const [codebasePath, localInfo] of Object.entries(localSnapshot.codebases)) { + const existingInfo = mergedCodebases[codebasePath]; + if (!existingInfo || this.toTimestamp(localInfo.lastUpdated) >= this.toTimestamp(existingInfo.lastUpdated)) { + mergedCodebases[codebasePath] = localInfo; + } + } + + for (const [codebasePath, deletedAt] of this.pendingDeletes.entries()) { + const currentInfo = mergedCodebases[codebasePath]; + if (!currentInfo || this.toTimestamp(deletedAt) >= this.toTimestamp(currentInfo.lastUpdated)) { + delete mergedCodebases[codebasePath]; + } + } + + return { + formatVersion: 'v2', + codebases: mergedCodebases, + lastUpdated: new Date().toISOString() + }; + } + + private markCodebaseDeleted(codebasePath: string): void { + this.pendingDeletes.set(codebasePath, new Date().toISOString()); + } + + private clearPendingSaveTimer(): void { + if (this.pendingSaveTimer) { + clearTimeout(this.pendingSaveTimer); + this.pendingSaveTimer = null; + } + this.pendingSaveReason = null; + } + + private enqueueSave(reason: string): Promise { + this.saveQueue = this.saveQueue + .catch(() => { + // Keep queue alive even if prior save failed. + }) + .then(async () => { + await this.performSaveCodebaseSnapshot(reason); + }); + return this.saveQueue; + } + + public scheduleSaveCodebaseSnapshot(reason: string = 'scheduled', delayMs: number = this.saveDebounceMs): void { + this.pendingSaveReason = reason; + + if (delayMs <= 0) { + const immediateReason = this.pendingSaveReason || 'scheduled-immediate'; + this.pendingSaveReason = null; + void this.enqueueSave(immediateReason).catch((error: any) => { + console.error('[SNAPSHOT-DEBUG] Error during immediate scheduled snapshot save:', error); + }); + return; + } + + if (this.pendingSaveTimer) { + return; + } + + this.pendingSaveTimer = setTimeout(() => { + const scheduledReason = this.pendingSaveReason || 'scheduled'; + this.pendingSaveReason = null; + this.pendingSaveTimer = null; + void this.enqueueSave(scheduledReason).catch((error: any) => { + console.error('[SNAPSHOT-DEBUG] Error during scheduled snapshot save:', error); + }); + }, delayMs); + } + + public async flushScheduledSave(): Promise { + if (this.pendingSaveTimer) { + clearTimeout(this.pendingSaveTimer); + this.pendingSaveTimer = null; + const reason = this.pendingSaveReason || 'scheduled-flush'; + this.pendingSaveReason = null; + await this.enqueueSave(reason); + return; + } + + await this.saveQueue; } /** @@ -105,22 +556,27 @@ export class SnapshotManager { continue; } - // Store the complete info for this codebase - validCodebaseInfoMap.set(codebasePath, info); - if (info.status === 'indexed') { + // Store the complete info for indexed codebases + validCodebaseInfoMap.set(codebasePath, info); validIndexedCodebases.push(codebasePath); if ('indexedFiles' in info) { validFileCount.set(codebasePath, info.indexedFiles); } console.log(`[SNAPSHOT-DEBUG] Validated indexed codebase: ${codebasePath} (${info.indexedFiles || 'unknown'} files, ${info.totalChunks || 'unknown'} chunks)`); } else if (info.status === 'indexing') { - if ('indexingPercentage' in info) { - validIndexingCodebases.set(codebasePath, info.indexingPercentage); - } console.warn(`[SNAPSHOT-DEBUG] Found interrupted indexing codebase: ${codebasePath} (${info.indexingPercentage || 0}%). Treating as not indexed.`); - // Don't add to indexed - treat interrupted indexing as not indexed + // Interrupted indexing should not block future indexing attempts. + // Convert it into a failed status to preserve diagnostics while avoiding stale "indexing" locks. + const interruptedInfo: CodebaseInfoIndexFailed = { + status: 'indexfailed', + errorMessage: 'Indexing was interrupted (likely due to MCP restart). Please run index_codebase again.', + lastAttemptedPercentage: info.indexingPercentage, + lastUpdated: new Date().toISOString() + }; + validCodebaseInfoMap.set(codebasePath, interruptedInfo); } else if (info.status === 'indexfailed') { + validCodebaseInfoMap.set(codebasePath, info); console.warn(`[SNAPSHOT-DEBUG] Found failed indexing codebase: ${codebasePath}. Error: ${info.errorMessage}`); // Failed indexing codebases are not added to indexed or indexing lists // But we keep the info for potential retry @@ -129,7 +585,7 @@ export class SnapshotManager { // Restore state this.indexedCodebases = validIndexedCodebases; - this.indexingCodebases = new Map(); // Reset indexing codebases since they were interrupted + this.indexingCodebases = validIndexingCodebases; this.codebaseFileCount = validFileCount; this.codebaseInfoMap = validCodebaseInfoMap; } @@ -137,21 +593,14 @@ export class SnapshotManager { public getIndexedCodebases(): string[] { // Read from JSON file to ensure consistency and persistence try { - if (!fs.existsSync(this.snapshotFilePath)) { + const snapshot = this.readSnapshotFromDiskUnsafe(); + if (!snapshot) { return []; } - const snapshotData = fs.readFileSync(this.snapshotFilePath, 'utf8'); - const snapshot: CodebaseSnapshot = JSON.parse(snapshotData); - - if (this.isV2Format(snapshot)) { - return Object.entries(snapshot.codebases) - .filter(([_, info]) => info.status === 'indexed') - .map(([path, _]) => path); - } else { - // V1 format - return snapshot.indexedCodebases || []; - } + return Object.entries(snapshot.codebases) + .filter(([_, info]) => info.status === 'indexed') + .map(([codebasePath, _]) => codebasePath); } catch (error) { console.warn(`[SNAPSHOT-DEBUG] Error reading indexed codebases from file:`, error); // Fallback to memory if file reading fails @@ -162,29 +611,14 @@ export class SnapshotManager { public getIndexingCodebases(): string[] { // Read from JSON file to ensure consistency and persistence try { - if (!fs.existsSync(this.snapshotFilePath)) { + const snapshot = this.readSnapshotFromDiskUnsafe(); + if (!snapshot) { return []; } - const snapshotData = fs.readFileSync(this.snapshotFilePath, 'utf8'); - const snapshot: CodebaseSnapshot = JSON.parse(snapshotData); - - if (this.isV2Format(snapshot)) { - return Object.entries(snapshot.codebases) - .filter(([_, info]) => info.status === 'indexing') - .map(([path, _]) => path); - } else { - // V1 format - Handle both legacy array format and new object format - if (Array.isArray(snapshot.indexingCodebases)) { - // Legacy format: return the array directly - return snapshot.indexingCodebases; - } else if (snapshot.indexingCodebases && typeof snapshot.indexingCodebases === 'object') { - // New format: return the keys of the object - return Object.keys(snapshot.indexingCodebases); - } - } - - return []; + return Object.entries(snapshot.codebases) + .filter(([_, info]) => info.status === 'indexing') + .map(([codebasePath, _]) => codebasePath); } catch (error) { console.warn(`[SNAPSHOT-DEBUG] Error reading indexing codebases from file:`, error); // Fallback to memory if file reading fails @@ -202,28 +636,14 @@ export class SnapshotManager { public getIndexingProgress(codebasePath: string): number | undefined { // Read from JSON file to ensure consistency and persistence try { - if (!fs.existsSync(this.snapshotFilePath)) { + const snapshot = this.readSnapshotFromDiskUnsafe(); + if (!snapshot) { return undefined; } - const snapshotData = fs.readFileSync(this.snapshotFilePath, 'utf8'); - const snapshot: CodebaseSnapshot = JSON.parse(snapshotData); - - if (this.isV2Format(snapshot)) { - const info = snapshot.codebases[codebasePath]; - if (info && info.status === 'indexing') { - return info.indexingPercentage || 0; - } - return undefined; - } else { - // V1 format - Handle both legacy array format and new object format - if (Array.isArray(snapshot.indexingCodebases)) { - // Legacy format: if path exists in array, assume 0% progress - return snapshot.indexingCodebases.includes(codebasePath) ? 0 : undefined; - } else if (snapshot.indexingCodebases && typeof snapshot.indexingCodebases === 'object') { - // New format: return the actual progress percentage - return snapshot.indexingCodebases[codebasePath]; - } + const info = snapshot.codebases[codebasePath]; + if (info && info.status === 'indexing') { + return info.indexingPercentage || 0; } return undefined; @@ -239,6 +659,7 @@ export class SnapshotManager { */ public addIndexingCodebase(codebasePath: string, progress: number = 0): void { this.indexingCodebases.set(codebasePath, progress); + this.pendingDeletes.delete(codebasePath); // Also update codebaseInfoMap for v2 compatibility const info: CodebaseInfoIndexing = { @@ -255,6 +676,7 @@ export class SnapshotManager { public updateIndexingProgress(codebasePath: string, progress: number): void { if (this.indexingCodebases.has(codebasePath)) { this.indexingCodebases.set(codebasePath, progress); + this.pendingDeletes.delete(codebasePath); // Also update codebaseInfoMap for v2 compatibility const info: CodebaseInfoIndexing = { @@ -273,6 +695,7 @@ export class SnapshotManager { this.indexingCodebases.delete(codebasePath); // Also remove from codebaseInfoMap for v2 compatibility this.codebaseInfoMap.delete(codebasePath); + this.markCodebaseDeleted(codebasePath); } /** @@ -282,6 +705,7 @@ export class SnapshotManager { if (!this.indexedCodebases.includes(codebasePath)) { this.indexedCodebases.push(codebasePath); } + this.pendingDeletes.delete(codebasePath); if (fileCount !== undefined) { this.codebaseFileCount.set(codebasePath, fileCount); } @@ -305,6 +729,7 @@ export class SnapshotManager { this.codebaseFileCount.delete(codebasePath); // Also remove from codebaseInfoMap for v2 compatibility this.codebaseInfoMap.delete(codebasePath); + this.markCodebaseDeleted(codebasePath); } /** @@ -334,6 +759,7 @@ export class SnapshotManager { */ public setCodebaseIndexing(codebasePath: string, progress: number = 0): void { this.indexingCodebases.set(codebasePath, progress); + this.pendingDeletes.delete(codebasePath); // Remove from other states this.indexedCodebases = this.indexedCodebases.filter(path => path !== codebasePath); @@ -355,6 +781,8 @@ export class SnapshotManager { codebasePath: string, stats: { indexedFiles: number; totalChunks: number; status: 'completed' | 'limit_reached' } ): void { + this.pendingDeletes.delete(codebasePath); + // Add to indexed list if not already there if (!this.indexedCodebases.includes(codebasePath)) { this.indexedCodebases.push(codebasePath); @@ -384,6 +812,8 @@ export class SnapshotManager { errorMessage: string, lastAttemptedPercentage?: number ): void { + this.pendingDeletes.delete(codebasePath); + // Remove from other states this.indexedCodebases = this.indexedCodebases.filter(path => path !== codebasePath); this.indexingCodebases.delete(codebasePath); @@ -433,6 +863,7 @@ export class SnapshotManager { this.indexingCodebases.delete(codebasePath); this.codebaseFileCount.delete(codebasePath); this.codebaseInfoMap.delete(codebasePath); + this.markCodebaseDeleted(codebasePath); console.log(`[SNAPSHOT-DEBUG] Completely removed codebase from snapshot: ${codebasePath}`); } @@ -441,24 +872,23 @@ export class SnapshotManager { console.log('[SNAPSHOT-DEBUG] Loading codebase snapshot from:', this.snapshotFilePath); try { - if (!fs.existsSync(this.snapshotFilePath)) { + this.pendingDeletes.clear(); + this.migrateLegacySnapshotIfNeeded(); + + const snapshot = this.readSnapshotFromDiskUnsafe(); + if (!snapshot) { console.log('[SNAPSHOT-DEBUG] Snapshot file does not exist. Starting with empty codebase list.'); return; } - const snapshotData = fs.readFileSync(this.snapshotFilePath, 'utf8'); - const snapshot: CodebaseSnapshot = JSON.parse(snapshotData); - console.log('[SNAPSHOT-DEBUG] Loaded snapshot:', snapshot); - if (this.isV2Format(snapshot)) { - this.loadV2Format(snapshot); - } else { - this.loadV1Format(snapshot); - } + this.loadV2Format(snapshot); // Always save in v2 format after loading (migration) - this.saveCodebaseSnapshot(); + void this.saveCodebaseSnapshot('post-load-migration').catch((error: any) => { + console.error('[SNAPSHOT-DEBUG] Error persisting post-load migration snapshot:', error); + }); } catch (error: any) { console.error('[SNAPSHOT-DEBUG] Error loading snapshot:', error); @@ -466,41 +896,29 @@ export class SnapshotManager { } } - public saveCodebaseSnapshot(): void { - console.log('[SNAPSHOT-DEBUG] Saving codebase snapshot to:', this.snapshotFilePath); - - try { - // Ensure directory exists - const snapshotDir = path.dirname(this.snapshotFilePath); - if (!fs.existsSync(snapshotDir)) { - fs.mkdirSync(snapshotDir, { recursive: true }); - console.log('[SNAPSHOT-DEBUG] Created snapshot directory:', snapshotDir); - } - - // Build v2 format snapshot using the complete info map - const codebases: Record = {}; - - // Add all codebases from the info map - for (const [codebasePath, info] of this.codebaseInfoMap) { - codebases[codebasePath] = info; - } - - const snapshot: CodebaseSnapshotV2 = { - formatVersion: 'v2', - codebases: codebases, - lastUpdated: new Date().toISOString() - }; - - fs.writeFileSync(this.snapshotFilePath, JSON.stringify(snapshot, null, 2)); - - const indexedCount = this.indexedCodebases.length; - const indexingCount = this.indexingCodebases.size; - const failedCount = this.getFailedCodebases().length; - - console.log(`[SNAPSHOT-DEBUG] Snapshot saved successfully in v2 format. Indexed: ${indexedCount}, Indexing: ${indexingCount}, Failed: ${failedCount}`); + private async performSaveCodebaseSnapshot(reason: string): Promise { + console.log(`[SNAPSHOT-DEBUG] Saving codebase snapshot to: ${this.snapshotFilePath} (reason=${reason})`); + + const mergedSnapshot = await this.withSnapshotLock(async () => { + const existingSnapshot = this.readSnapshotFromDiskUnsafe(); + const localSnapshot = this.buildSnapshotFromMemory(); + const merged = this.mergeSnapshots(existingSnapshot, localSnapshot); + this.writeSnapshotToDiskUnsafe(merged); + this.applySnapshotToMemory(merged); + this.pendingDeletes.clear(); + return merged; + }); + + const statuses = Object.values(mergedSnapshot.codebases); + const indexedCount = statuses.filter((info) => info.status === 'indexed').length; + const indexingCount = statuses.filter((info) => info.status === 'indexing').length; + const failedCount = statuses.filter((info) => info.status === 'indexfailed').length; + + console.log(`[SNAPSHOT-DEBUG] Snapshot saved successfully in v2 format. Indexed: ${indexedCount}, Indexing: ${indexingCount}, Failed: ${failedCount}`); + } - } catch (error: any) { - console.error('[SNAPSHOT-DEBUG] Error saving snapshot:', error); - } + public async saveCodebaseSnapshot(reason: string = 'manual'): Promise { + this.clearPendingSaveTimer(); + await this.enqueueSave(reason); } -} \ No newline at end of file +} diff --git a/packages/mcp/src/sync.ts b/packages/mcp/src/sync.ts index 0f53bab8..5b144ada 100644 --- a/packages/mcp/src/sync.ts +++ b/packages/mcp/src/sync.ts @@ -70,6 +70,17 @@ export class SyncManager { totalStats.modified += stats.modified; if (stats.added > 0 || stats.removed > 0 || stats.modified > 0) { + // Refresh snapshot metadata so get_indexing_status shows recent incremental reindex time. + const currentInfo = this.snapshotManager.getCodebaseInfo(codebasePath); + if (currentInfo && currentInfo.status === 'indexed') { + this.snapshotManager.setCodebaseIndexed(codebasePath, { + indexedFiles: currentInfo.indexedFiles, + totalChunks: currentInfo.totalChunks, + status: currentInfo.indexStatus + }); + await this.snapshotManager.saveCodebaseSnapshot('sync-incremental-updated'); + } + console.log(`[SYNC] Sync complete for '${codebasePath}'. Added: ${stats.added}, Removed: ${stats.removed}, Modified: ${stats.modified} (${codebaseElapsed}ms)`); } else { console.log(`[SYNC] No changes detected for '${codebasePath}' (${codebaseElapsed}ms)`); @@ -140,4 +151,4 @@ export class SyncManager { console.log('[SYNC-DEBUG] Background sync setup complete. Interval ID:', syncInterval); } -} \ No newline at end of file +}