diff --git a/src/cli.ts b/src/cli.ts index cc09329..a65c20e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -14,7 +14,6 @@ import readline from 'node:readline/promises' import { parseArgs } from 'node:util' import { Agent } from './agent.js' -import { estimateTokens } from './context.js' import { configFromEnv, type Config } from './config.js' import { LLM, LLMResponse, ScriptedLLM, type LLMClient } from './llm.js' import { StreamRenderer } from './render.js' @@ -324,9 +323,11 @@ async function handleCommand(input: string, agent: Agent, config: Config): Promi return true } if (input === '/compact') { - const before = estimateTokens(agent.messages) + // use the calibrated measure (fixed overhead + observed ratio), not the + // raw char estimate, so the numbers match what compression actually sees + const before = agent.context.measure(agent.messages) const compressed = await agent.context.maybeCompress(agent.messages, agent.llm) - const after = estimateTokens(agent.messages) + const after = agent.context.measure(agent.messages) if (compressed) { console.log(green(`Compressed: ${before} → ${after} tokens (${agent.messages.length} messages)`)) } else { diff --git a/src/tools/glob.ts b/src/tools/glob.ts index b2502b8..94a5888 100644 --- a/src/tools/glob.ts +++ b/src/tools/glob.ts @@ -47,10 +47,19 @@ export function globToRegExp(pattern: string): RegExp { const SKIP_DIRS = new Set(['.git', 'node_modules', '__pycache__', '.venv', 'venv', 'dist', 'build']) const MAX_WALK = 20_000 -async function walk(root: string): Promise> { - const results: Array<{ rel: string; abs: string; mtime: number }> = [] +async function walk( + root: string, +): Promise<{ files: Array<{ rel: string; abs: string; mtime: number }>; truncated: boolean }> { + const files: Array<{ rel: string; abs: string; mtime: number }> = [] const stack = [''] - while (stack.length > 0 && results.length < MAX_WALK) { + let truncated = false + while (stack.length > 0) { + if (files.length >= MAX_WALK) { + // stop walking but tell the caller the result is incomplete — a silent + // cap would look like the repo simply has no more matching files + truncated = true + break + } const relDir = stack.pop()! const absDir = path.join(root, relDir) let entries @@ -70,11 +79,11 @@ async function walk(root: string): Promise regex.test(f.rel)) + const { files, truncated } = await walk(base) + const hits = files.filter(f => regex.test(f.rel)) // sort by mtime, newest first hits.sort((a, b) => b.mtime - a.mtime) @@ -108,6 +118,7 @@ export const globTool: Tool = { const total = hits.length const shown = hits.slice(0, 100) let result = shown.map(h => h.abs).join('\n') + if (truncated) result += `\n... (search capped at ${MAX_WALK} files, results truncated)` if (total > 100) result += `\n... (${total} matches, showing first 100)` return result || 'No files matched.' } catch (e) { diff --git a/src/tools/grep.ts b/src/tools/grep.ts index 30ec912..74687f3 100644 --- a/src/tools/grep.ts +++ b/src/tools/grep.ts @@ -20,11 +20,21 @@ const SKIP_DIRS = new Set([ const MAX_FILES = 5000 const MAX_MATCHES = 200 -async function collectFiles(root: string, include?: string): Promise { +async function collectFiles( + root: string, + include?: string, +): Promise<{ files: string[]; truncated: boolean }> { const includeRe = include ? globToRegExp(include.includes('/') ? include : `**/${include}`) : null - const results: string[] = [] + const files: string[] = [] const stack = [''] - while (stack.length > 0 && results.length < MAX_FILES) { + let truncated = false + while (stack.length > 0) { + if (files.length >= MAX_FILES) { + // stop collecting but tell the caller — a silent cap would make a + // huge tree look like it simply has nothing matching + truncated = true + break + } const relDir = stack.pop()! let entries try { @@ -37,12 +47,11 @@ async function collectFiles(root: string, include?: string): Promise { if (entry.isDirectory()) { if (!SKIP_DIRS.has(entry.name)) stack.push(rel) } else if (entry.isFile()) { - if (!includeRe || includeRe.test(rel)) results.push(path.join(root, rel)) - if (results.length >= MAX_FILES) break + if (!includeRe || includeRe.test(rel)) files.push(path.join(root, rel)) } } } - return results + return { files, truncated } } export const grepTool: Tool = { @@ -74,7 +83,9 @@ export const grepTool: Tool = { const stat = await fs.stat(base).catch(() => null) if (!stat) return `Error: ${searchPath} not found` - const files = stat.isFile() ? [base] : await collectFiles(base, include) + const { files, truncated } = stat.isFile() + ? { files: [base], truncated: false } + : await collectFiles(base, include) const matches: string[] = [] for (const fp of files) { @@ -97,6 +108,12 @@ export const grepTool: Tool = { } } + if (truncated) { + // say the search was capped even when nothing matched, so the model + // doesn't conclude "no matches" from an incomplete search + if (matches.length === 0) return `No matches found (search capped at ${MAX_FILES} files).` + matches.push(`... (search capped at ${MAX_FILES} files, results truncated)`) + } return matches.length > 0 ? matches.join('\n') : 'No matches found.' }, } diff --git a/tests/search-tools.test.ts b/tests/search-tools.test.ts new file mode 100644 index 0000000..7490b47 --- /dev/null +++ b/tests/search-tools.test.ts @@ -0,0 +1,44 @@ +/** + * Search tool behavior: normal-path regression tests for glob/grep after + * the truncation-visibility change. (The truncation branches themselves need + * >20k files to trigger, so they're covered by code review, not tests.) + */ + +import assert from 'node:assert/strict' +import { promises as fs } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { test } from 'node:test' + +import { globTool } from '../src/tools/glob.js' +import { grepTool } from '../src/tools/grep.js' + +test('glob tool matches a small tree without truncation noise', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'cc-glob-')) + await fs.writeFile(path.join(dir, 'a.ts'), 'x') + await fs.writeFile(path.join(dir, 'b.js'), 'y') + await fs.mkdir(path.join(dir, 'sub')) + await fs.writeFile(path.join(dir, 'sub', 'c.ts'), 'z') + try { + const out = await globTool.execute({ pattern: '**/*.ts', path: dir }) + assert.ok(out.includes('a.ts'), 'top-level match found') + assert.ok(out.includes(path.join('sub', 'c.ts')), 'nested match found') + assert.ok(!out.includes('b.js'), 'non-matching extension excluded') + assert.ok(!out.includes('truncated'), 'no truncation note on a small tree') + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +}) + +test('grep tool searches a small tree and reports the match', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'cc-grep-')) + await fs.writeFile(path.join(dir, 'f.txt'), 'hello world\nnothing here\n') + try { + const out = await grepTool.execute({ pattern: 'hello', path: dir }) + assert.ok(out.includes('f.txt:1'), 'match reports path and line number') + assert.ok(out.includes('hello world'), 'match line content present') + assert.ok(!out.includes('truncated'), 'no truncation note on a small tree') + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } +})