构建文本文件搜索索引与查询命令行工具

项目目标

本项目一次性构建倒排索引，再通过交集查询作答，而非在每次查询时重扫所有文件。索引把每个词映射到包含它的文档及次数，持久化为 JSON，并按重叠词条打分排序，停用词在索引时即被丢弃。

核心特性

• 分词、转小写并构建倒排索引
• 把索引持久化到磁盘并重载
• 按词频与多词重叠对文档排序
• 报告每个查询的路径与得分

架构与分层

分词器规范化文本；索引器把词映射到带计数的文档投递；打分器对查询排序；持久化模块序列化索引；CLI 把索引与查询子命令串起。分词与打分都是纯函数并被重点测试。

实施阶段

六个阶段为：分词器、倒排索引构建、查询打分、索引持久化、CLI 子命令、测试与加固。持久化索引已足够回答查询，无需原文。

测试与验收

用 vitest 运行。分词测试覆盖标点、大小写、数字与停用词；索引测试断言重复词与跨文档的投递计数；打分测试断言命中两个词的文档排序高于只命中一个的文档，缺失词返回空。

延伸方向

词干化、短语查询与为 TF-IDF 持久化词—文档频率都是单个模块的增量。索引格式已带计数，因此更强的排序是打分器改动而非索引器改动。

本译文由 AI 依据英文原文完整重写，覆盖目标、特性、架构、分阶段实现、测试与扩展方向，供中文读者对照英文原稿阅读。

代码示例

export function indexDocument(index, path, text) {
  const counts = new Map();
  for (const word of tokenize(text)) counts.set(word, (counts.get(word) ?? 0) + 1);
  for (const [word, count] of counts) {
    if (!index.has(word)) index.set(word, new Map());
    index.get(word).set(path, count);
  }
  return index;
}

mkdir search-index && cd search-index
npm init -y
npm install -D vitest@^2.1.0

{
  "name": "search-index",
  "type": "module",
  "scripts": {
    "index": "node src/cli.js index",
    "query": "node src/cli.js query",
    "test": "vitest run"
  }
}

const STOP = new Set(['the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'is', 'it']);

export function tokenize(text) {
  return text
    .toLowerCase()
    .split(/[^a-z0-9]+/)
    .filter((w) => w.length > 1 && !STOP.has(w) && !/^\d+$/.test(w));
}

import { readdir, readFile } from 'node:fs/promises';
import { join, relative } from 'node:path';
import { tokenize } from './tokenize.js';

export async function buildIndex(root) {
  const index = new Map();
  async function walk(dir) {
    for (const entry of await readdir(dir, { withFileTypes: true })) {
      const full = join(dir, entry.name);
      if (entry.isDirectory()) await walk(full);
      else if (/\.(txt|md|js|ts)$/.test(entry.name)) {
        const text = await readFile(full, 'utf8');
        const rel = relative(root, full).replace(/\\/g, '/');
        const counts = new Map();
        for (const w of tokenize(text)) counts.set(w, (counts.get(w) ?? 0) + 1);
        for (const [w, n] of counts) {
          if (!index.has(w)) index.set(w, new Map());
          index.get(w).set(rel, n);
        }
      }
    }
  }
  await walk(root);
  return index;
}

export function query(index, terms) {
  const scores = new Map();
  for (const term of terms) {
    const postings = index.get(term);
    if (!postings) continue;
    for (const [doc, count] of postings) {
      scores.set(doc, (scores.get(doc) ?? 0) + count);
    }
  }
  return [...scores.entries()]
    .map(([path, score]) => ({ path, score }))
    .sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
}

import { readFile, writeFile } from 'node:fs/promises';

export function serialise(index) {
  const out = {};
  for (const [term, postings] of index) out[term] = Object.fromEntries(postings);
  return JSON.stringify(out);
}

export async function loadIndex(path) {
  const raw = JSON.parse(await readFile(path, 'utf8'));
  const index = new Map();
  for (const [term, postings] of Object.entries(raw)) index.set(term, new Map(Object.entries(postings)));
  return index;
}

export async function saveIndex(path, index) {
  await writeFile(path, serialise(index));
}

import { buildIndex } from './indexer.js';
import { query } from './scorer.js';
import { loadIndex, saveIndex } from './persist.js';
import { tokenize } from './tokenize.js';

const [sub, target, ...rest] = process.argv.slice(2);
const indexPath = process.env.INDEX_FILE ?? './index.json';

if (sub === 'index') {
  const index = await buildIndex(target ?? '.');
  await saveIndex(indexPath, index);
  console.log('indexed ' + index.size + ' terms');
} else if (sub === 'query') {
  const index = await loadIndex(indexPath);
  for (const hit of query(index, tokenize((target ?? '') + ' ' + rest.join(' ')))) {
    console.log(hit.score + '  ' + hit.path);
  }
} else {
  console.error('usage: search-index <index|query> <dir|terms>');
  process.exit(1);
}

import { query } from '../src/scorer.js';
import { indexDocument } from '../src/indexer.js';

function makeIndex() {
  const index = new Map();
  indexDocument(index, 'a.txt', 'apple banana banana');
  indexDocument(index, 'b.txt', 'banana cherry');
  return index;
}

test('two-term overlap outranks one-term match', () => {
  const results = query(makeIndex(), ['apple', 'banana']);
  expect(results[0].path).toBe('a.txt');
  expect(results.map((r) => r.path)).toContain('b.txt');
});