diff --git a/cli/schemas/output.schema.json b/cli/schemas/output.schema.json index 049cc48..8de54a9 100644 --- a/cli/schemas/output.schema.json +++ b/cli/schemas/output.schema.json @@ -374,6 +374,7 @@ "additionalProperties": false, "required": [ "query", + "searchMode", "results", "pagination" ], @@ -382,6 +383,13 @@ "type": "string", "minLength": 1 }, + "searchMode": { + "type": "string", + "enum": [ + "lexical", + "cjk-bigram-fallback" + ] + }, "results": { "type": "array", "items": { @@ -859,12 +867,19 @@ "type": "object", "additionalProperties": false, "required": [ - "created" + "created", + "next" ], "properties": { "created": { "type": "string", "minLength": 1 + }, + "next": { + "type": [ + "string", + "null" + ] } } }, diff --git a/cli/src/cli.ts b/cli/src/cli.ts index f0ab795..10b7485 100644 --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -113,7 +113,7 @@ export async function runCli(argv: string[], cwd = process.cwd(), stdin = ""): P program .command("search") - .description("按词法检索 llmdoc 文档(front matter、标题与正文,返回 snippet)") + .description("按词法检索 llmdoc 文档(自动中文分词，必要时使用 CJK bigram 降级)") .argument("", "检索词") .option("--topic ", "限定 topic") .option("--kind ", "限定类型: architecture | guide | reference") @@ -192,6 +192,10 @@ export async function runCli(argv: string[], cwd = process.cwd(), stdin = ""): P program .command("init-state") .description("首次生成 llmdoc/meta.json 台账骨架(validatedRevision 全部为 null)") + .addHelpText( + "after", + "\n前置: Git HEAD 必须已有真实 commit。生成后先 validate，再用 commit --all 完成 bootstrap。" + ) .action(async () => { const { runInitState } = await import("./commands/init-state.js"); const rootDir = findProjectRoot(cwd); @@ -230,14 +234,17 @@ export async function runCli(argv: string[], cwd = process.cwd(), stdin = ""): P .argument("", "目标相对路径,如 api-client/retry-policy.mdx") .requiredOption("--kind ", "文档类型: architecture | guide | reference") .option("--description ", "front matter 一句话描述") + .addHelpText( + "after", + "\n首次创建时会自动建立 llmdoc/；完成初始文档后运行 init-state → validate → commit --all。" + ) .action((targetPath, commandOptions) => { - const rootDir = findProjectRoot(cwd); output.push( writeOutput( "new", runNew({ ...globalOptions, - cwd: rootDir, + cwd, path: targetPath, kind: commandOptions.kind, description: commandOptions.description diff --git a/cli/src/commands/init-state.ts b/cli/src/commands/init-state.ts index 937e1fc..4a09406 100644 --- a/cli/src/commands/init-state.ts +++ b/cli/src/commands/init-state.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import { CliError } from "../lib/errors.js"; import { findProjectRoot } from "../lib/fs.js"; +import { isUnbornHead } from "../lib/git.js"; import { readWorkspaceGitState } from "../lib/state.js"; import { loadWorkspace } from "../lib/workspace.js"; import { MetaLedger } from "../types.js"; @@ -21,8 +22,16 @@ export function runInitState(options: InitStateOptions): unknown { throw new CliError("llmdoc/meta.json 已存在;init-state 只用于首次建立台账,不覆盖现有状态。"); } const git = readWorkspaceGitState(workspace); - if (!git.available || !git.headRevision) { - throw new CliError(git.degradedReason ?? "无法解析 HEAD commit,init-state 需要 git 仓库。"); + if (!git.available) { + throw new CliError("init-state 需要 Git 仓库；请先运行 `git init` 并创建一次真实的初始提交。"); + } + if (!git.headRevision) { + if (isUnbornHead(rootDir)) { + throw new CliError( + "HEAD 尚无 commit（当前分支尚未创建首次提交）。请先创建一次真实的初始提交；空仓库可运行 `git commit --allow-empty -m \"chore: initial commit\"`。" + ); + } + throw new CliError(`${git.degradedReason ?? "无法解析 HEAD commit。"}请先修复 Git HEAD，再运行 init-state。`); } const now = new Date().toISOString().replace(/\.\d+Z$/, "Z"); @@ -47,8 +56,9 @@ export function runInitState(options: InitStateOptions): unknown { status: "success", documents: workspace.documents.length, baselineRevision: git.headRevision, - next: "npx @tokenroll/llmdoc fingerprint --all" + next: + "npx -y @tokenroll/llmdoc validate && npx -y @tokenroll/llmdoc commit --all -m \"docs: bootstrap llmdoc\"" }; } - return `initialized llmdoc/meta.json: ${workspace.documents.length} documents (validatedRevision: null), baseline ${git.headRevision.slice(0, 7)}\nnext: 验证文档内容后运行 \`npx @tokenroll/llmdoc fingerprint --all\` 烙印 revision`; + return `initialized llmdoc/meta.json: ${workspace.documents.length} documents (validatedRevision: null), baseline ${git.headRevision.slice(0, 7)}\nnext: 先运行 \`npx -y @tokenroll/llmdoc validate\`；全部通过后运行 \`npx -y @tokenroll/llmdoc commit --all -m "docs: bootstrap llmdoc"\` 收尾`; } diff --git a/cli/src/commands/new.ts b/cli/src/commands/new.ts index b9b9ff2..915e076 100644 --- a/cli/src/commands/new.ts +++ b/cli/src/commands/new.ts @@ -3,7 +3,7 @@ import path from "node:path"; import { parseDocTargetShape, assertDocKindMatchesShape, assertDocumentKind } from "../lib/doc-shape.js"; import { CliError } from "../lib/errors.js"; -import { ensureDirectory, findProjectRoot, normalizeRepoRelativePath, resolveInsideRoot } from "../lib/fs.js"; +import { ensureDirectory, findProjectRootForNew, normalizeRepoRelativePath, resolveInsideRoot } from "../lib/fs.js"; import { packageRootFromImport } from "../lib/package-root.js"; import { loadWorkspace } from "../lib/workspace.js"; import { DocumentKind } from "../types.js"; @@ -17,7 +17,7 @@ interface NewOptions { } export function runNew(options: NewOptions): unknown { - const rootDir = findProjectRoot(options.cwd); + const rootDir = findProjectRootForNew(options.cwd); const repoRelativePath = normalizeDocDestination(options.path); const kind = assertDocumentKind(options.kind); const shape = parseDocTargetShape(repoRelativePath); @@ -37,15 +37,21 @@ export function runNew(options: NewOptions): unknown { .replace("__KIND__", () => kind) .replace("__TITLE__", () => title); + const metaExists = fs.existsSync(path.join(rootDir, "llmdoc", "meta.json")); fs.writeFileSync(absolutePath, content); syncMetaEntry(rootDir, shape.llmdocPath); if (options.json) { return { - created: repoRelativePath + created: repoRelativePath, + next: metaExists + ? null + : "若 HEAD 尚无 commit，请先创建首次 Git 提交；然后运行 `npx -y @tokenroll/llmdoc init-state`。" }; } - return `created: ${repoRelativePath}`; + return metaExists + ? `created: ${repoRelativePath}` + : `created: ${repoRelativePath}\nnext: 若仓库尚无提交，请先创建首次 Git 提交；然后运行 \`npx -y @tokenroll/llmdoc init-state\` 建立台账。`; } function normalizeDocDestination(input: string): string { diff --git a/cli/src/commands/search.ts b/cli/src/commands/search.ts index 61bb187..1ee2b2c 100644 --- a/cli/src/commands/search.ts +++ b/cli/src/commands/search.ts @@ -3,7 +3,7 @@ import { assertDocumentKind } from "../lib/doc-shape.js"; import { loadWorkspace } from "../lib/workspace.js"; import { OutputOptions } from "../types.js"; import { formatPaginationSummary } from "../lib/format.js"; -import { searchDocuments } from "../lib/search.js"; +import { SearchResult, searchDocuments } from "../lib/search.js"; import { estimateTokens } from "../lib/markdown.js"; interface SearchOptions extends OutputOptions { @@ -16,12 +16,12 @@ interface SearchOptions extends OutputOptions { export function runSearch(options: SearchOptions): unknown { const workspace = loadWorkspace(options.cwd); const kind = options.kind ? assertDocumentKind(options.kind) : undefined; - const results = searchDocuments(workspace, options.query, { + const search = searchDocuments(workspace, options.query, { topic: options.topic, kind }); const paginated = paginate({ - items: results, + items: search.results, estimate: (entry) => estimateTokens(JSON.stringify(toPayload(entry))), options }); @@ -29,12 +29,16 @@ export function runSearch(options: SearchOptions): unknown { if (options.json) { return { query: options.query, + searchMode: search.mode, results: paginated.items.map(toPayload), pagination: paginationMetadata(paginated) }; } const lines: string[] = []; + if (search.mode === "cjk-bigram-fallback") { + lines.push("note: 中文分词未命中，已使用 CJK bigram 降级检索。", ""); + } for (const entry of paginated.items) { lines.push(`llmdoc/${entry.document.llmdocPath} [${entry.document.frontmatter.kind}]`); lines.push(` ${entry.document.frontmatter.description}`); @@ -45,7 +49,7 @@ export function runSearch(options: SearchOptions): unknown { return lines.join("\n"); } -function toPayload(entry: ReturnType[number]): object { +function toPayload(entry: SearchResult): object { return { path: `llmdoc/${entry.document.llmdocPath}`, kind: entry.document.frontmatter.kind, diff --git a/cli/src/lib/fs.ts b/cli/src/lib/fs.ts index 8cdcdeb..59e818a 100644 --- a/cli/src/lib/fs.ts +++ b/cli/src/lib/fs.ts @@ -11,6 +11,21 @@ export function findProjectRoot(startDir: string): string { throw new CliError("未找到 llmdoc/ 目录，请在仓库内运行该命令。", 2); } +// new 是唯一允许在 llmdoc/ 尚不存在时运行的结构改写命令。 +// 首次创建严格锁定最近 Git 根；无 Git 但已有 llmdoc/ 时保留原有兼容路径。 +export function findProjectRootForNew(startDir: string): string { + const start = path.resolve(startDir); + const existingWorkspace = findProjectRootOrNull(start); + if (existingWorkspace) { + return existingWorkspace; + } + const gitRoot = findNearestGitRootOrNull(start); + if (gitRoot) { + return gitRoot; + } + throw new CliError("未找到 Git 仓库；请先运行 `git init`，再运行 `llmdoc new`。", 2); +} + export function findProjectRootOrNull(startDir: string): string | null { const start = path.resolve(startDir); const gitRoot = findNearestGitRootOrNull(start); diff --git a/cli/src/lib/git.ts b/cli/src/lib/git.ts index 1cb035b..f9c7bd0 100644 --- a/cli/src/lib/git.ts +++ b/cli/src/lib/git.ts @@ -35,6 +35,13 @@ export function gitCommitExists(rootDir: string, revision: string): boolean { return result.status === 0; } +export function isUnbornHead(rootDir: string): boolean { + if (!isGitRepository(rootDir) || runGitSafe(rootDir, ["rev-parse", "--verify", "HEAD"]) !== null) { + return false; + } + return runGitSafe(rootDir, ["symbolic-ref", "--quiet", "HEAD"]) !== null; +} + // shallow clone(CI 常态)里历史 commit 不可达,revision 校验需要据此降级而不是误报陈旧。 export function isShallowRepository(rootDir: string): boolean { return runGitSafe(rootDir, ["rev-parse", "--is-shallow-repository"]) === "true"; @@ -56,7 +63,7 @@ export function readGitState(rootDir: string, baselineRevision: string | null): }; } - const headRevision = runGitSafe(rootDir, ["rev-parse", "HEAD"]); + const headRevision = runGitSafe(rootDir, ["rev-parse", "--verify", "HEAD"]); const detached = runGitSafe(rootDir, ["symbolic-ref", "--quiet", "--short", "HEAD"]) === null; const inProgressOperation = detectInProgressOperation(rootDir); const stagedPaths = readPathList(rootDir, ["diff", "--name-only", "--no-renames", "--cached"]); @@ -73,7 +80,7 @@ export function readGitState(rootDir: string, baselineRevision: string | null): let degradedReason: string | null = null; if (!headRevision) { - degradedReason = "无法解析 HEAD commit。"; + degradedReason = isUnbornHead(rootDir) ? "HEAD 尚无 commit（当前分支尚未创建首次提交）。" : "无法解析 HEAD commit。"; } else if (baselineRevision && !gitCommitExists(rootDir, baselineRevision)) { degradedReason = `baseline.revision 不存在于当前 git 历史: ${baselineRevision}`; } diff --git a/cli/src/lib/search.ts b/cli/src/lib/search.ts index c13655f..cb4f1b2 100644 --- a/cli/src/lib/search.ts +++ b/cli/src/lib/search.ts @@ -9,10 +9,11 @@ interface SearchCacheEntry { llmdocPath: string; mtimeMs: number; searchableText: string; + wordCount: number; } interface SearchCacheFile { - version: 1; + version: 2; entries: Record; } @@ -22,12 +23,52 @@ export interface SearchResult { snippet: string; } +export type SearchMode = "lexical" | "cjk-bigram-fallback"; + +export interface SearchResponse { + results: SearchResult[]; + mode: SearchMode; +} + +interface SearchToken { + value: string; + weight: number; +} + +const WORD_SEGMENTER = new Intl.Segmenter("zh-CN", { granularity: "word" }); +const CJK_CHARACTER = /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]$/u; +const CJK_RUN = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]{2,}/gu; +const QUERY_STOP_WORDS = new Set([ + "的", + "了", + "呢", + "吗", + "啊", + "吧", + "是", + "在", + "与", + "和", + "或", + "及", + "把", + "被", + "从", + "到", + "为", + "怎么", + "如何", + "什么", + "为何" +]); + export function searchDocuments( workspace: WorkspaceData, query: string, filters?: { topic?: string; kind?: string } -): SearchResult[] { - const tokens = tokenize(query); +): SearchResponse { + const normalizedQuery = query.trim().toLowerCase(); + const lexicalTokens = tokenizeQuery(query).map((value) => ({ value, weight: 1 })); const cache = loadSearchCache(workspace); const filteredDocs = workspace.documents.filter((document) => { if (filters?.topic && document.topic !== filters.topic) { @@ -39,41 +80,95 @@ export function searchDocuments( return true; }); + const lexicalResults = scoreDocuments(filteredDocs, cache, lexicalTokens, { + minimumMatchedTokens: 1, + normalizedQuery + }); + if (lexicalResults.length > 0) { + return { + results: lexicalResults, + mode: "lexical" + }; + } + + const bigrams = cjkBigrams(query); + if (bigrams.length === 0) { + return { + results: [], + mode: "lexical" + }; + } + const minimumMatchedTokens = bigrams.length >= 4 ? Math.max(2, Math.ceil(bigrams.length * 0.25)) : 1; + return { + results: scoreDocuments( + filteredDocs, + cache, + bigrams.map((value) => ({ value, weight: 0.55 })), + { minimumMatchedTokens, normalizedQuery } + ), + mode: "cjk-bigram-fallback" + }; +} + +function scoreDocuments( + documents: ParsedDocument[], + cache: SearchCacheFile, + tokens: SearchToken[], + options: { minimumMatchedTokens: number; normalizedQuery: string } +): SearchResult[] { + if (tokens.length === 0) { + return []; + } + const documentFrequency = new Map(); for (const token of tokens) { let count = 0; - for (const document of filteredDocs) { + for (const document of documents) { const searchableText = cache.entries[document.llmdocPath]?.searchableText ?? buildSearchableText(document); - if (searchableText.includes(token)) { + if (searchableText.includes(token.value)) { count += 1; } } - documentFrequency.set(token, count); + documentFrequency.set(token.value, count); } - return filteredDocs + return documents .map((document) => { const searchableText = cache.entries[document.llmdocPath]?.searchableText ?? buildSearchableText(document); - const wordCount = tokenize(searchableText).length || 1; + const wordCount = cache.entries[document.llmdocPath]?.wordCount ?? countWords(searchableText); let score = 0; + let matchedTokenCount = 0; for (const token of tokens) { - const frequency = countSubstring(searchableText, token); + const frequency = countSubstring(searchableText, token.value); if (frequency === 0) { continue; } - const df = documentFrequency.get(token) ?? 0; - const idf = Math.log(1 + (filteredDocs.length - df + 0.5) / (df + 0.5)); - score += idf * ((frequency * 2.2) / (frequency + 1.2 * (1 - 0.75 + 0.75 * (wordCount / 200)))); + matchedTokenCount += 1; + const df = documentFrequency.get(token.value) ?? 0; + const idf = Math.log(1 + (documents.length - df + 0.5) / (df + 0.5)); + score += + token.weight * + idf * + ((frequency * 2.2) / (frequency + 1.2 * (1 - 0.75 + 0.75 * (wordCount / 200)))); + } + + if (options.normalizedQuery.length > 1) { + const phraseFrequency = countSubstring(searchableText, options.normalizedQuery); + score += phraseFrequency * (tokens.length + 1) * 2; } return { document, score, - snippet: buildSnippet(document.body, tokens) + matchedTokenCount, + snippet: buildSnippet( + document.body, + tokens.map((token) => token.value) + ) }; }) - .filter((entry) => entry.score > 0) + .filter((entry) => entry.score > 0 && entry.matchedTokenCount >= options.minimumMatchedTokens) .sort((left, right) => right.score - left.score || left.document.llmdocPath.localeCompare(right.document.llmdocPath)); } @@ -98,7 +193,7 @@ function normalizeCodePathPattern(pattern: string): string { function loadSearchCache(workspace: WorkspaceData): SearchCacheFile { const next: SearchCacheFile = { - version: 1, + version: 2, entries: {} }; const cachePath = resolveCachePath(workspace); @@ -107,14 +202,16 @@ function loadSearchCache(workspace: WorkspaceData): SearchCacheFile { for (const document of workspace.documents) { const stats = fs.statSync(document.absolutePath); const cachedEntry = existing?.entries[document.llmdocPath]; - if (cachedEntry && cachedEntry.mtimeMs === stats.mtimeMs) { + if (existing?.version === 2 && cachedEntry && cachedEntry.mtimeMs === stats.mtimeMs) { next.entries[document.llmdocPath] = cachedEntry; continue; } + const searchableText = buildSearchableText(document); next.entries[document.llmdocPath] = { llmdocPath: document.llmdocPath, mtimeMs: stats.mtimeMs, - searchableText: buildSearchableText(document) + searchableText, + wordCount: countWords(searchableText) }; } @@ -158,12 +255,60 @@ function buildSearchableText(document: ParsedDocument): string { .toLowerCase(); } -function tokenize(input: string): string[] { - return input - .toLowerCase() - .split(/[^a-z0-9\u4e00-\u9fff_-]+/) - .map((token) => token.trim()) - .filter(Boolean); +function tokenizeQuery(input: string): string[] { + const tokens: string[] = []; + let singleCjkBuffer: string[] = []; + const flushSingleCjkBuffer = (): void => { + if (singleCjkBuffer.length > 0) { + tokens.push(singleCjkBuffer.join("")); + singleCjkBuffer = []; + } + }; + + for (const part of WORD_SEGMENTER.segment(input.toLowerCase())) { + const value = part.segment.trim(); + if (!part.isWordLike || !value) { + flushSingleCjkBuffer(); + continue; + } + if (QUERY_STOP_WORDS.has(value)) { + flushSingleCjkBuffer(); + continue; + } + if (CJK_CHARACTER.test(value)) { + singleCjkBuffer.push(value); + continue; + } + flushSingleCjkBuffer(); + tokens.push(value); + } + flushSingleCjkBuffer(); + return unique(tokens); +} + +function cjkBigrams(input: string): string[] { + const bigrams: string[] = []; + for (const match of input.toLowerCase().matchAll(CJK_RUN)) { + const characters = [...match[0]]; + for (let index = 0; index < characters.length - 1; index += 1) { + bigrams.push(`${characters[index]}${characters[index + 1]}`); + } + } + return unique(bigrams); +} + +function countWords(input: string): number { + let count = 0; + for (const part of WORD_SEGMENTER.segment(input)) { + if (part.isWordLike && part.segment.trim()) { + count += 1; + } + } + return count || 1; +} + +function unique(values: string[]): string[] { + return [...new Set(values)]; } function buildSnippet(body: string, tokens: string[]): string { diff --git a/cli/src/lib/workspace.ts b/cli/src/lib/workspace.ts index 1cb323c..6f95981 100644 --- a/cli/src/lib/workspace.ts +++ b/cli/src/lib/workspace.ts @@ -164,7 +164,8 @@ export function validateWorkspace(workspace: WorkspaceData): ValidationIssue[] { severity: "error", code: "meta.missing", path: "llmdoc/meta.json", - message: "缺少 llmdoc/meta.json。" + message: + "缺少 llmdoc/meta.json。请运行 `npx -y @tokenroll/llmdoc init-state` 建立台账（若仓库尚无提交，请先创建首次 Git 提交）。" }); } diff --git a/cli/tests/cold-start.test.ts b/cli/tests/cold-start.test.ts new file mode 100644 index 0000000..5df7512 --- /dev/null +++ b/cli/tests/cold-start.test.ts @@ -0,0 +1,119 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { describe, expect, test } from "vitest"; + +import { runCli } from "../src/cli.js"; + +describe("cold-start workflow", () => { + test("new creates the first llmdoc directory at the nearest Git root", async () => { + const rootDir = createEmptyGitRepository(); + const nestedDir = path.join(rootDir, "src", "nested"); + fs.mkdirSync(nestedDir, { recursive: true }); + + const created = await runCli( + ["new", "architecture.mdx", "--kind", "architecture", "--description", "仓库整体架构。"], + nestedDir + ); + + expect(created.exitCode).toBe(0); + expect(created.stdout).toContain("created: llmdoc/architecture.mdx"); + expect(created.stdout).toContain("init-state"); + expect(fs.existsSync(path.join(rootDir, "llmdoc", "architecture.mdx"))).toBe(true); + + const jsonCreated = await runCli( + ["--json", "new", "runtime/overview.mdx", "--kind", "reference", "--description", "运行时边界。"], + nestedDir + ); + const payload = JSON.parse(jsonCreated.stdout) as { created: string; next: string | null }; + expect(payload.created).toBe("llmdoc/runtime/overview.mdx"); + expect(payload.next).toContain("init-state"); + }); + + test("new refuses a directory without a Git repository with an actionable remedy", async () => { + const plainDir = fs.mkdtempSync(path.join(os.tmpdir(), "llmdoc-no-git-")); + + const result = await runCli(["new", "architecture.mdx", "--kind", "architecture"], plainDir); + + expect(result.exitCode).toBe(2); + expect(result.stdout).toContain("git init"); + expect(fs.existsSync(path.join(plainDir, "llmdoc"))).toBe(false); + }); + + test("new preserves the existing non-Git llmdoc compatibility path", async () => { + const plainDir = fs.mkdtempSync(path.join(os.tmpdir(), "llmdoc-existing-no-git-")); + fs.mkdirSync(path.join(plainDir, "llmdoc")); + + const result = await runCli(["new", "architecture.mdx", "--kind", "architecture"], plainDir); + + expect(result.exitCode).toBe(0); + expect(fs.existsSync(path.join(plainDir, "llmdoc", "architecture.mdx"))).toBe(true); + }); + + test("init-state explains unborn HEAD and then exposes the validated bootstrap sequence", async () => { + const rootDir = createEmptyGitRepository(); + const created = await runCli(["new", "architecture.mdx", "--kind", "architecture"], rootDir); + expect(created.exitCode).toBe(0); + + const missingMeta = await runCli(["validate"], rootDir); + expect(missingMeta.exitCode).toBe(1); + expect(missingMeta.stdout).toContain("init-state"); + + const unborn = await runCli(["init-state"], rootDir); + expect(unborn.exitCode).toBe(1); + expect(unborn.stdout).toContain("HEAD 尚无 commit"); + expect(unborn.stdout).toContain("git commit --allow-empty"); + + commitEmpty(rootDir); + const initialized = await runCli(["--json", "init-state"], rootDir); + const payload = JSON.parse(initialized.stdout) as { status: string; next: string }; + expect(initialized.exitCode).toBe(0); + expect(payload.status).toBe("success"); + expect(payload.next).toContain("validate"); + expect(payload.next).toContain("commit --all"); + expect(payload.next).not.toContain("fingerprint"); + + const validated = await runCli(["validate"], rootDir); + expect(validated.exitCode).toBe(0); + }); + + test("new synchronizes an existing ledger without repeating the bootstrap hint", async () => { + const rootDir = createEmptyGitRepository(); + await runCli(["new", "architecture.mdx", "--kind", "architecture"], rootDir); + commitEmpty(rootDir); + await runCli(["init-state"], rootDir); + + const created = await runCli( + ["--json", "new", "runtime/details.mdx", "--kind", "guide", "--description", "运行细节。"], + rootDir + ); + const payload = JSON.parse(created.stdout) as { created: string; next: string | null }; + expect(created.exitCode).toBe(0); + expect(payload.next).toBeNull(); + + const meta = JSON.parse(fs.readFileSync(path.join(rootDir, "llmdoc", "meta.json"), "utf8")) as { + documents: Record; + }; + expect(meta.documents["runtime/details.mdx"]).toEqual({ validatedRevision: null }); + }); +}); + +function createEmptyGitRepository(): string { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "llmdoc-cold-start-")); + runGit(rootDir, ["init", "--quiet"]); + runGit(rootDir, ["config", "user.email", "test@example.com"]); + runGit(rootDir, ["config", "user.name", "Test User"]); + return rootDir; +} + +function commitEmpty(rootDir: string): void { + runGit(rootDir, ["commit", "--allow-empty", "--quiet", "-m", "initial commit"]); +} + +function runGit(rootDir: string, args: string[]): void { + const result = spawnSync("git", args, { cwd: rootDir, encoding: "utf8" }); + if (result.status !== 0) { + throw new Error((result.stderr || result.stdout || `git ${args[0]} failed`).trim()); + } +} diff --git a/cli/tests/search-cjk.test.ts b/cli/tests/search-cjk.test.ts new file mode 100644 index 0000000..89714d2 --- /dev/null +++ b/cli/tests/search-cjk.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, test } from "vitest"; + +import { runCli } from "../src/cli.js"; +import { createFixture, writeRepoFile } from "./helpers.js"; + +interface SearchPayload { + searchMode: "lexical" | "cjk-bigram-fallback"; + results: Array<{ path: string; score: number }>; +} + +describe("CJK search", () => { + test("segments natural Chinese queries without requiring manual spaces", async () => { + const rootDir = createFixture(); + writeRepoFile( + rootDir, + "llmdoc/mock-world/traffic-control.mdx", + `--- +description: 终端流量按权重分摊，限速由令牌桶生效。 +kind: guide +--- + +# 流量控制 + +终端流量按权重分摊。系统通过令牌桶实施限速，配置保存后立即生效。 +` + ); + + const naturalQuestion = await runCli(["--json", "search", "限速是怎么生效的"], rootDir); + const naturalPayload = JSON.parse(naturalQuestion.stdout) as SearchPayload; + expect(naturalQuestion.exitCode).toBe(0); + expect(naturalPayload.searchMode).toBe("lexical"); + expect(naturalPayload.results[0]?.path).toBe("llmdoc/mock-world/traffic-control.mdx"); + + const conceptQuery = await runCli(["--json", "search", "终端速率分摊"], rootDir); + const conceptPayload = JSON.parse(conceptQuery.stdout) as SearchPayload; + expect(conceptQuery.exitCode).toBe(0); + expect(conceptPayload.searchMode).toBe("lexical"); + expect(conceptPayload.results[0]?.path).toBe("llmdoc/mock-world/traffic-control.mdx"); + }); + + test("prefers exact phrases over partial lexical matches", async () => { + const rootDir = createFixture(); + writeRepoFile( + rootDir, + "llmdoc/ranking/exact.mdx", + `--- +description: 终端速率分摊的精确说明。 +kind: reference +--- + +# 精确命中 + +终端速率分摊。 +` + ); + writeRepoFile( + rootDir, + "llmdoc/ranking/partial.mdx", + `--- +description: 终端流量的分摊说明。 +kind: reference +--- + +# 部分命中 + +终端流量按权重分摊。 +` + ); + + const result = await runCli(["--json", "search", "终端速率分摊"], rootDir); + const payload = JSON.parse(result.stdout) as SearchPayload; + expect(payload.results[0]?.path).toBe("llmdoc/ranking/exact.mdx"); + expect(payload.results[0]!.score).toBeGreaterThan(payload.results[1]!.score); + }); + + test("uses an annotated bigram fallback when lexical terms have no hits", async () => { + const rootDir = createFixture(); + writeRepoFile( + rootDir, + "llmdoc/devices/throughput.mdx", + `--- +description: 吞吐限制的配置规则。 +kind: reference +--- + +# 吞吐限制 + +系统通过配额设置吞吐限制。 +` + ); + + const jsonResult = await runCli(["--json", "search", "吞吐量"], rootDir); + const payload = JSON.parse(jsonResult.stdout) as SearchPayload; + expect(jsonResult.exitCode).toBe(0); + expect(payload.searchMode).toBe("cjk-bigram-fallback"); + expect(payload.results[0]?.path).toBe("llmdoc/devices/throughput.mdx"); + + const textResult = await runCli(["search", "吞吐量"], rootDir); + expect(textResult.stdout).toContain("已使用 CJK bigram 降级检索"); + + const noisyLongQuery = await runCli(["--json", "search", "吞吐量完全未知"], rootDir); + const noisyPayload = JSON.parse(noisyLongQuery.stdout) as SearchPayload; + expect(noisyPayload.searchMode).toBe("cjk-bigram-fallback"); + expect(noisyPayload.results).toEqual([]); + }); + + test("keeps existing single-term Chinese and Latin search behavior", async () => { + const rootDir = createFixture(); + + const chinese = await runCli(["--json", "search", "重试"], rootDir); + const chinesePayload = JSON.parse(chinese.stdout) as SearchPayload; + expect(chinesePayload.searchMode).toBe("lexical"); + expect(chinesePayload.results.some((entry) => entry.path === "llmdoc/api-client/retry-policy.mdx")).toBe(true); + + const latin = await runCli(["--json", "search", "retry"], rootDir); + const latinPayload = JSON.parse(latin.stdout) as SearchPayload; + expect(latinPayload.searchMode).toBe("lexical"); + expect(latinPayload.results.some((entry) => entry.path === "llmdoc/api-client/retry-policy.mdx")).toBe(true); + }); +}); diff --git a/docs/v3-design/03-cli.md b/docs/v3-design/03-cli.md index 82f65e7..7535b6e 100644 --- a/docs/v3-design/03-cli.md +++ b/docs/v3-design/03-cli.md @@ -19,10 +19,10 @@ CLI 是 V3 的 Runtime 实体:所有确定性、可测试、重复出现的工 | `llmdoc tree` | **根入口的正式替代**(V2 `index.md` 的能力由它承担):输出根单例 + 全部 topic 及其文档名聚合摘要(topic 无入口节点)。默认紧凑到 topic 级;`--docs` 展开到文档级(path/kind/description)。L0/L1 | | `llmdoc index [--topic t] [--kind k]` | 批量输出文档 front matter 投影(path/description/kind/relations/code.paths)。L2 | | `llmdoc show ` | 按路径取正文,多文档合并输出,带预算。L3 | -| `llmdoc search [--topic] [--kind]` | 词法检索(front matter + 标题 + 正文,BM25 级),返回 path + description + 命中片段 | +| `llmdoc search [--topic] [--kind]` | 词法检索(front matter + 标题 + 正文,BM25 级);中文查询先分词，零 lexical 召回时降级为 CJK bigram，返回 path + description + 命中片段 | | `llmdoc context --files ` | **给 AI 的核心入口**:"我要改这些源码文件,应该先读哪些文档"——逐输入用 `code.paths` 反查 + requires 闭包,并独立报告 `unmappedFiles` | -搜索索引缓存于 `.llmdoc-tmp/cache/`,按 mtime/revision 增量重建,删除可再生。不做 embedding。 +搜索索引缓存于 `.llmdoc-tmp/cache/`,按 mtime/revision 增量重建,删除可再生。不做 embedding。完整短语优先于部分词项；发生 CJK bigram 降级时，文本输出会明确提示，JSON 的 `searchMode` 会暴露实际检索模式。 仓库根可选 `.llmdocignore`(每行一个 minimatch pattern,`#` 注释,`dir/` 自动展开为 `dir/**`):匹配路径不参与 unmapped/dirty 信号,用于本地运行时文件、数据库等非知识面路径。 @@ -34,7 +34,7 @@ CLI 是 V3 的 Runtime 实体:所有确定性、可测试、重复出现的工 | `llmdoc delta [--scope ]` | 变更代码 → 受影响文档闭包 + unmapped paths + light/deep 建议信号(见 04) | | `llmdoc validate` | 全量校验:front matter schema、kind 合法、禁 index.mdx、层级深度(禁嵌套)、链接/requires 悬空、CodeRef path 存在、`code.paths` 精确路径存在且 glob 至少命中一项、ledger 与文件树一致、体积告警。CI 与写入门控共用 | | `llmdoc fingerprint --update ` | 只刷新 revision 的低层台账原语;普通 update 收尾优先用 `commit`/`commit --verified` 保证 validate 与 git 提交闭环 | -| `llmdoc init-state` | 首次生成 `meta.json` 台账骨架:全部文档 `validatedRevision: null` + 实测 convergence;init/upgrade 场景专用,拒绝覆盖已有台账 | +| `llmdoc init-state` | 在真实 Git HEAD 上首次生成 `meta.json` 台账骨架:全部文档 `validatedRevision: null` + 实测 convergence;unborn 仓库须先建立首次 commit,且拒绝覆盖已有台账 | | `llmdoc commit [-m] [--verified \| --all] [--no-verify]` | **一体化收尾**:validate 门控 → 可选提交正文写集(不卷入用户 staged 的其他文件)→ 刷新正文改动与 verified-unchanged 文档 → meta 单独小 commit。`--all` 表示全量复核且不能与 `--verified` 同用;无正文改动时可只提交 meta。消灭手工三步曲与 `--amend` 追尾陷阱 | ### 2.3 Hook 面(webhook 需要做的事全部收敛于此) @@ -53,7 +53,7 @@ Fail policy:hook 执行失败不阻塞开发;hook 永不写 `llmdoc/`;写命令 | 命令 | 作用 | |---|---| -| `llmdoc new --kind ` | 脚手架:生成带合法 front matter 的空文档 | +| `llmdoc new --kind ` | 脚手架:生成带合法 front matter 的空文档；最近 Git 根尚无 `llmdoc/` 时自动创建，并在缺少 ledger 时提示后续 `init-state` | | `llmdoc adopt ` | 无损登记:把已存在的合法 `.mdx` 登记进 `meta.json`(`validatedRevision: null`,不改正文,幂等) | | `llmdoc mv ` | 重命名/移动:`git mv` + 批量更新引用与 ledger key | | `llmdoc prune --report` | growth 报告:当前规模 vs convergence baseline、重复/碎片候选(只报告,收敛动作由 Recorder 做) | diff --git a/docs/v3-design/04-workflows.md b/docs/v3-design/04-workflows.md index 18db452..50c0f8c 100644 --- a/docs/v3-design/04-workflows.md +++ b/docs/v3-design/04-workflows.md @@ -68,7 +68,10 @@ preflight: 确认 llmdoc/ 不存在(存在有效 V3 → 拒绝并建议 update;V → 多个 Investigator 按互补主题调查(能力/数据流/集成/构建发布/横切约定) → coverage 检查与 follow-up(补缺口、解决冲突) → Recorder bootstrap:定 topic 集合 → 写根 architecture.mdx + 各 topic 按需文档(无入口节点) -→ llmdoc validate 门控 → 生成 meta.json(baseline + convergence) +→ unborn Git 仓库先建立真实 HEAD commit +→ llmdoc init-state 生成 meta.json(baseline + convergence) +→ llmdoc validate + Context Floor 验收 +→ llmdoc commit --all 收尾 ``` 要点:先定 topic 边界再写正文;**不生成根 index**(地图由 `llmdoc tree` 动态承担);只创建支撑高价值知识的文档,不求全(自动生成的大而全入口已被实践证明有害);不生成空 folder。 diff --git a/llmdoc/cli-runtime/retrieval-and-mutation.mdx b/llmdoc/cli-runtime/retrieval-and-mutation.mdx index ed38702..3800f6d 100644 --- a/llmdoc/cli-runtime/retrieval-and-mutation.mdx +++ b/llmdoc/cli-runtime/retrieval-and-mutation.mdx @@ -12,6 +12,7 @@ code: - cli/src/commands/index.ts - cli/src/commands/show.ts - cli/src/commands/search.ts + - cli/src/lib/search.ts - cli/src/commands/context.ts - cli/src/commands/new.ts - cli/src/commands/adopt.ts @@ -31,6 +32,8 @@ code: - cli/tests/viewer-state.test.ts - cli/tests/viewer-http.test.ts - cli/tests/viewer-assets.test.ts + - cli/tests/search-cjk.test.ts + - cli/tests/cold-start.test.ts - cli/schemas/output.schema.json --- @@ -38,18 +41,16 @@ code: ## 渐进读取模型 -读取面按“地图/索引或搜索/正文”逐层暴露，调用者在任一层都可以停止；这些命令返回受预算约束的公开投影，不泄漏内部解析对象。`context --files` 对每个输入独立用 `code.paths` 反查 owner，再对 owner 并集补齐 `relations.requires` 前置闭包；未命中的输入通过 `unmappedFiles` 单独返回，不能被同批查询中的成功命中掩盖。文本与 JSON 输出共用 schema 契约，避免不同宿主形成第二套语义。 +读取面按“地图/索引或搜索/正文”渐进披露，任一层都允许停止。`context --files` 对每个输入独立反查 owner，再为 owner 并集补齐 `requires` 前置闭包；未命中输入必须单独返回，不能被同批成功项掩盖。文本与 JSON 输出共用语义契约，避免宿主分叉。 -Viewer 是同一知识面的只读本地投影，不是新的改写入口。服务命令只负责 loopback 生命周期与装配；HTTP 层只接受 GET/HEAD 和显式资产/API 路由，文档读取必须命中 workspace 扫描得到的 canonical ID，不能把请求路径解释成文件系统路径；状态层把 workspace、delta、validate 与 growth 投影为可序列化 DTO，并以 `requires > related > link` 的优先级确定同向关系，反向关系仍独立保留。 +`search` 先做中文词法分段，使连续 CJK 查询不依赖人工空格；词法零召回时才降级到确定性的 CJK bigram。降级必须在文本与 JSON 中显式暴露，不能把弱匹配伪装成正常召回；完整短语优先，bigram 只承担兜底。 -浏览器端保持无框架的原生模块边界：model 负责视图模型，graph 负责确定性布局与交互，detail 负责详情导航和安全渲染，app 只协调状态与 UI。服务端防御性响应头与客户端 Markdown 清洗共同构成渲染边界；任何扩展都不能绕过路由白名单、canonical 文档查找或在未清洗时注入 HTML。 +Viewer 是同一知识面的 loopback 只读投影，不是改写入口。请求只能经过显式路由并命中 workspace 扫描出的 canonical 文档 ID，不能解释为任意文件系统路径；服务端响应边界与客户端 Markdown 清洗必须同时保留，扩展也不得绕过。 ## 结构改写不变量 -所有路径先经过仓库与 `llmdoc/` realpath 边界校验，符号链接逃逸和覆盖现有目标都会被拒绝。`adopt` 只登记已有合法正文；`mv` 负责移动及内部引用/ledger key 重写，失败时回滚，不触碰源码。 +所有结构改写都受仓库与 `llmdoc/` realpath 边界约束，拒绝符号链接逃逸和覆盖目标。`new` 是唯一可在缺少 `llmdoc/` 时启动的改写，它锁定最近 Git 根；其他改写要求已有 workspace，失败必须原子回滚且不触碰源码。 -`commit` 是正式收尾边界：先 validate 和 revision/dirty 预检，再只提交 `llmdoc/` 写集，最后刷新 fingerprint 并单独提交 `meta.json`。`--verified` 把已复核但无需改正文的文档纳入 fingerprint，可形成 meta-only commit；有正文改动时，changed 与 verified 文档取并集。`--all` 表示全量复核，不能与 `--verified` 同用。 - -预检失败必须在创建任何 commit 前 fail-closed，用户已 staged 的非 llmdoc 文件不得被卷入。fingerprint 锚定正文提交前后的真实 HEAD，因此不得用 `--amend` 把 meta 追进前一提交。 +`commit` 是正式收尾边界：先 validate 和 revision/dirty 预检，只提交 `llmdoc/` 写集，再刷新 fingerprint 并单独提交 `meta.json`；已复核但正文未变的 owner 也可纳入 fingerprint。预检失败必须在任何 commit 前 fail-closed，不能卷入用户已 staged 的非 llmdoc 文件，也不得用 amend 合并 meta follow-up。 `prune` 与 `upgrade` 的 CLI 只负责可重复的诊断；合并、删除、迁移和知识价值判断始终由工作流层交给 `recorder`。 diff --git a/llmdoc/cli-runtime/state-and-validation.mdx b/llmdoc/cli-runtime/state-and-validation.mdx index 5f4cd51..d71b02d 100644 --- a/llmdoc/cli-runtime/state-and-validation.mdx +++ b/llmdoc/cli-runtime/state-and-validation.mdx @@ -12,6 +12,7 @@ code: - cli/src/lib/schema.ts - cli/src/lib/viewer-state.ts - cli/src/commands/validate.ts + - cli/src/commands/init-state.ts - cli/src/commands/status.ts - cli/src/commands/delta.ts - cli/src/commands/fingerprint.ts @@ -19,6 +20,7 @@ code: - cli/tests/cli.test.ts - cli/tests/viewer-state.test.ts - cli/tests/viewer-http.test.ts + - cli/tests/cold-start.test.ts - cli/schemas/doc-frontmatter.schema.json - cli/schemas/meta.schema.json - cli/schemas/output.schema.json @@ -28,9 +30,11 @@ code: ## Workspace 与文档模型 -CLI 只接受最近 Git 根直属的 `llmdoc/`，避免跨仓库误认；没有 Git 边界时才保留初始化场景的向上兼容查找。V3 文档树固定为根 singleton 加一层 topic，路径即文档 ID；不设 `index.mdx`，topic 摘要由 CLI 聚合。`meta.json` 只保存有效性台账，不复制可扫描的树结构。 +除首次 `new` 外，CLI 只接受最近 Git 根直属的已有 `llmdoc/`，避免跨仓库误认；没有 Git 边界时仅兼容查找已有知识目录。V3 文档树固定为根 singleton 加一层 topic，路径即 ID；topic 没有入口文档，`meta.json` 也只保存有效性台账，不复制可扫描结构。 -`validate` 的责任是保证 schema、两层结构、引用、代码锚点、realpath 边界与 ledger/文件树一致；`code.paths` 的精确路径必须存在，glob 也必须至少命中一个现有文件，避免陈旧 pattern 静默通过。它只证明结构有效，不判断某个 pattern 是否错误吸附无关文件，或 first-class owner 是否缺失；这些语义问题属于 Routing Gate 与 Context Floor 验收。精确字段集合属于 schema，不在稳定正文重复维护。 +`validate` 保证结构、引用、代码锚点、realpath 边界与 ledger 一致，并拒绝不存在的精确路径或空匹配 glob。它只证明结构有效，不判断 glob 是否吸附无关文件或 first-class owner 是否缺失；这些语义问题属于 Routing Gate 与 Context Floor。 + +`init-state` 的 baseline 必须锚定真实 HEAD；unborn branch 先创建首次提交。Git、HEAD 或 ledger 前置失败必须给出对应 remedy，不能折叠成模糊错误。 ## Revision 与影响语义 @@ -38,7 +42,7 @@ CLI 只接受最近 Git 根直属的 `llmdoc/`，避免跨仓库误认；没有 `delta` 先用 `code.paths` 找直接受影响文档，再沿 `relations.requires` 反向扩展一跳为 needs-review。未映射路径、反向复核、影响过广、dirty 关联代码或失效 revision 会把建议模式抬到 deep。命中表示必须复核，不表示正文必须变化。 -revision 推进前必须确认 git 可推进且目标文档关联实现无 dirty；全量推进还要求所有实现表面 clean。`commit --verified` 让语义仍成立的文档只刷新有效性锚点。 +revision 推进前必须确认 Git 可推进且目标 owner 的关联实现无 dirty；全量推进要求所有实现表面 clean。正文仍成立的 owner 只刷新有效性锚点。 `status` 与 Viewer 共用 repository health 判定：原始 commits-behind 保留为 Git 历史可观测值，知识陈旧度只计其中触碰 implementation surface 的 relevant commit。behind 全为 metadata-only 时仍是 knowledge-clean，不能直接把 `behindHead > 0` 渲染成 stale；无法可靠计算 relevant 数量时必须保持 unknown/degraded，而不是乐观判 fresh。fingerprint 之后由 CLI 自己创建的纯 `llmdoc/meta.json` follow-up 因此不代表知识再次过期，重复收尾应成为 no-op。 diff --git a/llmdoc/meta.json b/llmdoc/meta.json index 7c5669f..d3085b7 100644 --- a/llmdoc/meta.json +++ b/llmdoc/meta.json @@ -6,28 +6,28 @@ }, "documents": { "architecture.mdx": { - "validatedRevision": "255d8672b5c4de14149324c15a017392b0865488" + "validatedRevision": "d5c2ecc1c6de581c4469072274f313b39673ec4b" }, "cli-runtime/state-and-validation.mdx": { - "validatedRevision": "255d8672b5c4de14149324c15a017392b0865488" + "validatedRevision": "9b57dba1d1f05ae503780b3bd0de634a41e93d57" }, "cli-runtime/retrieval-and-mutation.mdx": { - "validatedRevision": "2b114c1e18652ea1ef337848f5837a67157bf057" + "validatedRevision": "9b57dba1d1f05ae503780b3bd0de634a41e93d57" }, "plugin-packaging/claude-and-codex.mdx": { - "validatedRevision": "255d8672b5c4de14149324c15a017392b0865488" + "validatedRevision": "9b57dba1d1f05ae503780b3bd0de634a41e93d57" }, "plugin-packaging/development-and-release.mdx": { - "validatedRevision": "255d8672b5c4de14149324c15a017392b0865488" + "validatedRevision": "9b57dba1d1f05ae503780b3bd0de634a41e93d57" }, "workflows/init-and-update.mdx": { - "validatedRevision": "2b114c1e18652ea1ef337848f5837a67157bf057" + "validatedRevision": "9b57dba1d1f05ae503780b3bd0de634a41e93d57" }, "workflows/prune-and-upgrade.mdx": { - "validatedRevision": "2b114c1e18652ea1ef337848f5837a67157bf057" + "validatedRevision": "d5c2ecc1c6de581c4469072274f313b39673ec4b" }, "website/product-and-deployment.mdx": { - "validatedRevision": "255d8672b5c4de14149324c15a017392b0865488" + "validatedRevision": "d5c2ecc1c6de581c4469072274f313b39673ec4b" } }, "convergence": { diff --git a/llmdoc/plugin-packaging/claude-and-codex.mdx b/llmdoc/plugin-packaging/claude-and-codex.mdx index 976d51a..9217e8e 100644 --- a/llmdoc/plugin-packaging/claude-and-codex.mdx +++ b/llmdoc/plugin-packaging/claude-and-codex.mdx @@ -24,16 +24,16 @@ code: ## 真相源与 runtime 启动边界 -Claude 根插件是手工维护的 canonical surface：skills 定义 Retrieval/Reflection/Stable Knowledge Gate 与显式工作流，三个 agent 定义调查、候选捕获和正式写入边界。Codex 的 skills、agents、manifest 与 marketplace 是转换产物，不得成为独立设计源。 +Claude 根插件是手工维护的 canonical surface；skills 定义知识门与显式工作流，agents 分隔调查、候选捕获和正式写入。Codex 的 skills、agents 与清单是转换产物，不得成为独立设计源。 -两种宿主都把确定性行为收敛到外部 `@tokenroll/llmdoc` CLI。普通交互命令使用 `npx -y @tokenroll/llmdoc `：不向消费仓库写依赖；需要复现时在包名中固定版本；不得退回会解析错误包名或交互挂起的裸调用。 +两种宿主都把确定性行为收敛到外部 `@tokenroll/llmdoc` CLI，不向消费仓库写依赖；需要复现时固定包版本，不能退回会解析错误包名的裸调用。 -生命周期 hook 属于不同的启动边界：宿主从消费仓库的 cwd 调用 npm，而该仓库可能暴露同名的 local/`file:` 依赖。npm 会让这个依赖先满足普通 package identity；即使其 bin 尚未构建或不是应运行的发布 runtime，进程也不会自动回退到 registry CLI。hook 因此必须通过一个不同 identity 的 scoped npm alias 请求 `@tokenroll/llmdoc`，再调用它暴露的 `llmdoc hook *` bin。alias 只隔离 package 解析，不改变共同 runtime、参数或输出契约，也不应用于日常交互命令。 +生命周期 hook 从消费仓库 cwd 启动，可能被同名 local/`file:` 依赖抢先满足，且无效 bin 不会自动回退 registry。hook 因此必须借助不同 identity 的 scoped npm alias 选择发布 runtime；alias 只隔离包解析，不改变 CLI 契约，也不用于日常交互。 ## 生成与一致性不变量 -转换必须在临时副本生成后替换式同步，因为生成器不保证清除陈旧输出。Codex 角色文本不手工维护；宿主特有 front matter/TOML 可以不同，实际 skill 与 agent 指令正文必须与 Claude canonical 一致。被 skill 按需加载的 reference 同样属于可执行 prompt surface：Codex 镜像必须包含相同正文，相关 skill/agent 也必须保留可发现的加载路径。 +转换必须在临时副本生成后替换式同步，防止陈旧输出残留。宿主特有元数据可以不同，skill、agent 及其按需 reference 的实际指令正文必须与 Claude canonical 一致并保持可发现。 -`scripts/check-codex-surface.mjs` 在 CI 中机械校验版本/marketplace 身份、五个 skill 与三个 agent 的正文一致性、按需 reference 的正文 parity 与加载路径、Reflection Gate 约束和三种 lifecycle hook 的 alias 调用。它还分别遍历 `skills/` 与 `.agents/skills/`，用 `gray-matter` 解析每份 `SKILL.md` 的 YAML front matter，并要求 `name`、`description` 是非空字符串；这是两侧各自的语法门槛，不替代跨宿主正文 parity。人工 checklist 只补宿主 UI policy、知识拓扑质量与信任模型等无法从正文等价判断的部分。 +CI 必须同时检查生成面身份、指令/reference parity、可发现路径、front matter 语法和 lifecycle alias；语法有效不能替代跨宿主正文一致。人工复核只补 UI policy、知识拓扑与信任模型等无法机械等价判断的部分。 hooks 位于共享插件根，一份配置供两种宿主使用；它们保持只读，且由安装宿主按自己的信任模型启用。CLI 成功启动后 hook 逻辑保持 fail-open，但进程启动前的 npm 解析或 registry 失败不受这层保护；保留 alias 启动边界是避免本地同名依赖把错误暴露在宿主生命周期中的必要条件。 diff --git a/llmdoc/plugin-packaging/development-and-release.mdx b/llmdoc/plugin-packaging/development-and-release.mdx index 98db742..5af9832 100644 --- a/llmdoc/plugin-packaging/development-and-release.mdx +++ b/llmdoc/plugin-packaging/development-and-release.mdx @@ -18,7 +18,7 @@ code: ## 消费与开发边界 -公开产物是 `@tokenroll/llmdoc` CLI；仓库根 package 只是私有开发工作区。消费项目通过外部 npx/global 安装使用 CLI，不把 llmdoc 写进自己的依赖和 lockfile。根工作区直接声明的 `gray-matter` 只供 `scripts/check-codex-surface.mjs` 解析 skill front matter，不是消费方依赖，也没有为 CLI 新增 runtime 依赖。 +公开产物是 `@tokenroll/llmdoc` CLI，仓库根 package 只是私有开发工作区。消费项目从外部使用 CLI，不把 llmdoc 写进自己的依赖和 lockfile；开发工作区依赖不得泄漏为 CLI runtime 依赖。 ## 非平凡编辑前先同步 @@ -26,9 +26,7 @@ code: ## 变更集同步不变量 -CLI 命令或输出语义变化必须在同一 change-set 同步：canonical skills/agents、生成的 Codex 表面、版本与 lockfile、双语 README/design docs，以及 dogfood `llmdoc/`。精确文件数量由自动化维护，不在正文复制。 - -`scripts/check-codex-surface.mjs` 负责版本、身份与生成正文 parity；CI 还运行 lint、typecheck、test、build、prompt budget、dogfood validate 和消费者安装 smoke。自动化通过后仍需审查语义是否在所有公开表面一致，特别是宿主 UI policy 与授权边界。 +CLI 命令或输出语义变化必须在同一 change-set 同步 canonical prompt、生成宿主表面、版本元数据、公开文档和 dogfood knowledge；精确文件集合交给自动化。机械校验通过后仍需人工审查宿主 policy、授权边界和公开语义是否一致。 ## Lifecycle hook 发布验证 diff --git a/llmdoc/workflows/init-and-update.mdx b/llmdoc/workflows/init-and-update.mdx index caf31ff..6d52053 100644 --- a/llmdoc/workflows/init-and-update.mdx +++ b/llmdoc/workflows/init-and-update.mdx @@ -14,7 +14,9 @@ code: - agents/recorder.md - cli/src/commands/status.ts - cli/src/commands/delta.ts + - cli/src/commands/init-state.ts - cli/src/commands/fingerprint.ts + - cli/tests/cold-start.test.ts - skills/llmdoc/references/knowledge-topology.md - .agents/skills/llmdoc/references/knowledge-topology.md --- @@ -23,20 +25,22 @@ code: ## 授权与角色边界 -两条工作流第一次正式写入前都要求 `llmdoc/` clean，写后必须 validate，失败且无法修复时回滚本次文档写集。它们只授权知识面与临时候选处理，不授权源码编辑。 +两条工作流正式写入前都要求 `llmdoc/` clean，写后必须 validate；无法修复就回滚本次知识写集。授权只覆盖知识面与临时候选，不包含源码。 `init` 只用于没有有效 V3 知识面的仓库；已有 V3 转 update，legacy 结构只能走显式 upgrade。首次知识面先以稳定职责、authority、failure model 与查询词汇识别 domain，再把它们组织为持久 retrieval topic，最后由 `recorder` 为每个决策簇或风险工作流建立 canonical owner。Package 与目录只是边界证据；root singleton 只承载真正跨 topic 的契约。 目标是最小充分 owner 集，不是源码全量转述或固定文档数量。每个 first-class 子系统都必须在 scratch domain/owner matrix 中归为 documented、intentionally reconstructable 或 gap；no-doc 决定需要可快速恢复的 canonical source 与无隐藏持久语义的证据，未决 gap 会阻止 init success。 +首次 bootstrap 在 `init-state` 前必须同时具备已完成的首批文档和真实 HEAD；unborn 仓库先创建一次真实初始提交，不能用工作树状态代替 revision。其后顺序固定为 `init-state` 建立 ledger、`validate` 与 Context Floor 验收、最后 `commit --all` 收尾；跳过或调换这些阶段会留下缺失台账、未验证路由或未闭合 fingerprint。 + ## Update 是语义复核 -入口由 `status`、`delta` 和显式 `--reflection` 候选共同组成。它们形成复核集合，不是正文写入列表： +入口状态、delta 和显式 reflection 候选只形成复核集合，不是正文写入列表： - light：owner 已映射、事实与根因清楚，`recorder` 可用定向证据判断； - deep：存在 unmapped、owner/根因不清、边界变化、冲突、影响过广或失效 revision，先由 `investigator` 调查。 -reflection candidate 先验证 trigger、错误动作、根因、预防规则、scope 与置信度；用户纠正证明意图，不自动证明仓库事实。通过质量门后仍要经过 Stable Knowledge Gate：只有会改变未来选择、难以快速重建、足够持久且归属明确的规则才进入既有 architecture/guide。瞬时失败、单任务偏好和原始证据留在临时队列。 +reflection candidate 先核实触发、根因、预防规则与适用范围；用户纠正证明意图，不自动证明仓库事实。只有会改变未来选择、难以快速重建、足够持久且归属明确的规则才折入既有 owner；瞬时失败、单任务偏好和原始证据留在临时队列。 代码 delta 同样只产生复核义务。原文失效或出现合格新结论才改写；原文仍成立时标记 verified unchanged，不追加本次 diff、路径库存或证据。 @@ -44,8 +48,8 @@ Stable Knowledge Gate 与 Routing Gate 独立：前者决定正文是否值得 ## 收尾与 revision -`recorder` 负责正文和 validate，calling workflow 负责 `commit`：正文改动与 `--verified` 文档可一次取 fingerprint 并集；全部未改时用 meta-only `commit --verified`；全仓复核才用 `--all`。相关实现仍 dirty 时不得推进 revision。 +`recorder` 负责正文和 validate，calling workflow 用 `commit` 统一收尾；改写 owner 与 verified-unchanged owner 可一起刷新 fingerprint，全仓复核才推进全量 baseline。相关实现 dirty 时不得推进 revision。 fingerprint 锚定真实 commit。rebase/squash 会改写 hash，使台账在 fresh clone 中失效，因此含 fingerprint 的提交应保留历史或在合并后重新全量烙印；不得用 amend 合并 meta follow-up。 -reflection 候选只有在成功晋升、确认已覆盖或明确驳回后才移入 resolved；incomplete/failed 保持 pending。工作流只报告既定五态，success 以语义复核、validate 和 commit 闭环为准。 +reflection 候选只有在成功晋升、确认已覆盖或明确驳回后才移入 resolved；未完成或失败时保持 pending。success 以语义复核、validate 和 commit 闭环为准。 diff --git a/llmdoc/workflows/prune-and-upgrade.mdx b/llmdoc/workflows/prune-and-upgrade.mdx index 88b14c3..64e565d 100644 --- a/llmdoc/workflows/prune-and-upgrade.mdx +++ b/llmdoc/workflows/prune-and-upgrade.mdx @@ -10,6 +10,7 @@ code: - skills/upgrade/SKILL.md - cli/src/commands/prune.ts - cli/src/commands/upgrade.ts + - cli/src/commands/init-state.ts --- # Prune 与 Upgrade diff --git a/website/src/pages/docs/cli/index.astro b/website/src/pages/docs/cli/index.astro index dd6bee3..f9d54ae 100644 --- a/website/src/pages/docs/cli/index.astro +++ b/website/src/pages/docs/cli/index.astro @@ -14,7 +14,7 @@ import DocsLayout from "../../../layouts/DocsLayout.astro";
tree
Dynamic map of root documents and topics.
index [--topic …] [--kind …]
Front-matter projection for document discovery.
-
search <query>
Lexical search across tracked knowledge.
+
search <query>
Lexical search across tracked knowledge, with Chinese word segmentation and an explicitly reported CJK-bigram fallback after zero lexical recall.
context --files <src…>
Map each source path to owner documents and required prerequisites, while reporting unmapped inputs explicitly.
show <path…>
Read selected document bodies.
serve
Start the local Viewer on 127.0.0.1 to inspect document structure, relations, and status.
@@ -29,9 +29,10 @@ import DocsLayout from "../../../layouts/DocsLayout.astro";

Mutate safely

-
new <path> --kind <kind>
Scaffold a V3 document.
+
new <path> --kind <kind>
Scaffold a V3 document, creating the first llmdoc/ directory at the nearest Git root when needed.
adopt <path…>
Register valid existing documents without rewriting their bodies.
mv <from> <to>
Move a document and rewrite internal references transactionally.
+
init-state
Seed the first ledger on a real Git HEAD; an unborn repository must create its initial commit first.
commit [--verified <path…> | --all]
Validate, commit changed prose, and refresh changed or verified revisions.
fingerprint --update <path…> | --all
Refresh validated revisions in the ledger.
diff --git a/website/src/pages/docs/workflows/index.astro b/website/src/pages/docs/workflows/index.astro index d0447ac..894ee95 100644 --- a/website/src/pages/docs/workflows/index.astro +++ b/website/src/pages/docs/workflows/index.astro @@ -20,7 +20,7 @@ import DocsLayout from "../../../layouts/DocsLayout.astro";

Initialize

Use the init workflow only when a repository has no valid V3 knowledge surface. It designs domain and topic boundaries, writes the smallest sufficient set of owner documents, and classifies every first-class subsystem as documented, intentionally reconstructable, or a blocking gap.

-

After structural validation, init verifies natural concept queries, representative files from every implementation boundary, wildcard negative probes, and prerequisite closure before it can finish with Git-backed state.

+

After drafting the initial documents, an unborn repository first creates a real HEAD commit. Then init-state seeds the ledger, validate checks structure, Context Floor probes verify routing, and commit --all completes the Git-backed bootstrap.

Update

Update semantically verifies existing knowledge against code deltas and authorized reflection candidates. A document that is still correct is verified unchanged; it does not accumulate a change log.

diff --git a/website/src/pages/zh/docs/cli/index.astro b/website/src/pages/zh/docs/cli/index.astro index 2e8521b..eee98a1 100644 --- a/website/src/pages/zh/docs/cli/index.astro +++ b/website/src/pages/zh/docs/cli/index.astro @@ -15,7 +15,7 @@ import DocsLayout from "../../../../layouts/DocsLayout.astro";
tree
根文档与 topic 的动态地图。
index [--topic …] [--kind …]
用于发现文档的 front matter 投影。
-
search <query>
在跟踪知识中执行词法搜索。
+
search <query>
在跟踪知识中执行词法搜索；中文查询自动分词，零 lexical 召回时明确标注 CJK bigram 降级。
context --files <src…>
逐个把源码路径映射到 owner 文档与依赖前置，并明确报告未映射输入。
show <path…>
读取选中的文档正文。
serve
在 127.0.0.1 启动本地 Viewer，查看文档结构、关系与状态。
@@ -30,9 +30,10 @@ import DocsLayout from "../../../../layouts/DocsLayout.astro";

安全改写

-
new <path> --kind <kind>
创建 V3 文档骨架。
+
new <path> --kind <kind>
创建 V3 文档骨架；最近 Git 根尚无 llmdoc/ 时会自动建立目录。
adopt <path…>
登记已有合法文档，不重写正文。
mv <from> <to>
事务式移动文档并重写内部引用。
+
init-state
在真实 Git HEAD 上建立首次 ledger；unborn 仓库须先创建初始 commit。
commit [--verified <path…> | --all]
校验、提交正文，并刷新变化或验证未变文档的 revision。
fingerprint --update <path…> | --all
刷新 ledger 中经过验证的 revision。
diff --git a/website/src/pages/zh/docs/workflows/index.astro b/website/src/pages/zh/docs/workflows/index.astro index 841a382..5e234ab 100644 --- a/website/src/pages/zh/docs/workflows/index.astro +++ b/website/src/pages/zh/docs/workflows/index.astro @@ -21,7 +21,7 @@ import DocsLayout from "../../../../layouts/DocsLayout.astro";

初始化

只有仓库不存在有效 V3 知识面时，才使用 init。它设计 domain 与 topic 边界、写入最小充分的 owner 文档，并把每个 first-class 子系统归类为“已文档化”“有意由源码重建”或阻塞缺口。

-

结构校验通过后，init 还要验证自然语言概念查询、每个 implementation boundary 的代表文件、wildcard 负例与必读关系闭包，才能以 Git 状态收尾。

+

首批文档完成后，unborn 仓库先创建真实 HEAD commit；随后由 init-state 建立 ledger、validate 校验结构、Context Floor 探针验证路由，最后用 commit --all 完成 Git 收尾。

更新

update 对照代码 delta 与已授权的 reflection candidate，语义复核现有知识。仍然正确的文档会被标记为 verified unchanged，不会累积变更日志。