构建 RSS 订阅聚合命令行工具

项目目标

本项目构建一个订阅聚合工具，并发抓取多个 RSS 与 Atom 源，归一化成统一结构，按链接去重并按时间倒序输出摘要或 JSON。抓取、解析与合并彼此分离，因此解析器可对着提交的样例文件离线测试。

核心特性

• 并发抓取多源并带单请求超时
• 把 RSS 与 Atom 解析为同一结构
• 按链接去重跨源重复文章
• 可读摘要与原始 JSON 两种输出

架构与分层

抓取层带超时与并发上限获取文档；归一化层把两种格式映射到同一字段；合并层去重并排序；格式化层输出文本或 JSON。解析与抓取分离，使样例可离线验证。

实施阶段

六个阶段为：订阅抓取、RSS 与 Atom 解析、合并与去重、输出格式化、CLI 表面、测试与加固。单源超时不得拖垮整轮。

测试与验收

用 vitest 运行。解析测试读取提交的样例断言两种格式的归一化字段，含无日期条目；合并测试用重叠条目断言按链接去重与倒序；抓取测试用桩替换全局 fetch 断言超时被报告为错误。

延伸方向

全文搜索、按源过滤与缓存都是既有管道外围的新阶段；新增格式是一次归一化，新增输出模式是一次格式化，均不改动抓取与合并。

本译文由 AI 依据英文原文完整重写，覆盖目标、特性、架构、分阶段实现、测试与扩展方向，供中文读者对照英文原稿阅读。

代码示例

export function normalizeRss(channel) {
  const items = asArray(channel.item);
  return items.map((item) => ({
    title: text(item.title),
    link: text(item.link),
    publishedAt: toIso(text(item.pubDate)),
    summary: text(item.description),
    source: text(channel.title),
  }));
}

export function normalizeAtom(feed) {
  const entries = asArray(feed.entry);
  return entries.map((entry) => ({
    title: text(entry.title),
    link: text(entry.link?.['@_href'] ?? entry.link),
    publishedAt: toIso(text(entry.updated ?? entry.published)),
    summary: text(entry.summary),
    source: text(feed.title),
  }));
}

mkdir feed-reader && cd feed-reader
npm init -y
npm install fast-xml-parser@^4.4.0
npm install -D vitest@^2.1.0

{
  "name": "feed-reader",
  "type": "module",
  "scripts": {
    "read": "node src/cli.js",
    "test": "vitest run"
  }
}

export async function fetchFeeds(urls, { timeoutMs = 8000, concurrency = 4 } = {}) {
  const results = [];
  let index = 0;
  async function worker() {
    while (index < urls.length) {
      const url = urls[index++];
      const controller = new AbortController();
      const timer = setTimeout(() => controller.abort(), timeoutMs);
      try {
        const res = await fetch(url, { signal: controller.signal });
        results.push({ url, text: await res.text() });
      } catch (err) {
        results.push({ url, error: err.name });
      } finally {
        clearTimeout(timer);
      }
    }
  }
  await Promise.all(Array.from({ length: Math.min(concurrency, urls.length) }, worker));
  return results;
}

import { XMLParser } from 'fast-xml-parser';

const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@_' });

export function parseFeed(xml) {
  const doc = parser.parse(xml);
  if (doc.rss?.channel) return normalizeRss(doc.rss.channel);
  if (doc.feed) return normalizeAtom(doc.feed);
  return [];
}

const asArray = (v) => (Array.isArray(v) ? v : v == null ? [] : [v]);
const text = (v) => (v == null ? '' : typeof v === 'object' ? v['#text'] ?? '' : String(v));

function toIso(value) {
  const d = new Date(value);
  return Number.isNaN(d.getTime()) ? null : d.toISOString();
}

export function mergeItems(lists) {
  const seen = new Map();
  for (const list of lists) {
    for (const item of list) {
      const key = item.link || item.title;
      if (!seen.has(key)) seen.set(key, item);
    }
  }
  return [...seen.values()].sort((a, b) => {
    const da = a.publishedAt ?? '';
    const db = b.publishedAt ?? '';
    if (da !== db) return db.localeCompare(da);
    return a.title.localeCompare(b.title);
  });
}

export function formatDigest(items, { limit = 20 } = {}) {
  return items.slice(0, limit).map((item, i) => {
    const date = item.publishedAt ? item.publishedAt.slice(0, 10) : 'unknown';
    return `${String(i + 1).padStart(2)}. ${item.title} (${item.source}, ${date})\n    ${item.link}`;
  }).join('\n');
}

import { fetchFeeds } from './fetchFeeds.js';
import { parseFeed } from './parseFeed.js';
import { mergeItems } from './merge.js';
import { formatDigest } from './format.js';

const urls = process.argv.slice(2).filter((a) => !a.startsWith('--'));
const asJson = process.argv.includes('--json');
if (urls.length === 0) {
  console.error('usage: read <feed-url> [more...] [--json]');
  process.exit(1);
}
const fetched = await fetchFeeds(urls);
for (const f of fetched.filter((f) => f.error)) console.error('failed: ' + f.url + ' (' + f.error + ')');
const items = mergeItems(fetched.filter((f) => f.text).map((f) => parseFeed(f.text)));
console.log(asJson ? JSON.stringify(items, null, 2) : formatDigest(items));

import { mergeItems } from '../src/merge.js';

test('duplicate links collapse and newest sorts first', () => {
  const a = [{ link: 'https://x/1', title: 'One', publishedAt: '2026-01-01T00:00:00Z' }];
  const b = [
    { link: 'https://x/1', title: 'One (cross-post)', publishedAt: '2026-01-01T00:00:00Z' },
    { link: 'https://x/2', title: 'Two', publishedAt: '2026-02-01T00:00:00Z' },
  ];
  const merged = mergeItems([a, b]);
  expect(merged.map((m) => m.link)).toEqual(['https://x/2', 'https://x/1']);
});