湖仓一体

工作原理

湖仓一体把数据文件（通常是 Parquet）保存在对象存储中，再由表格式增加元数据层，例如事务日志或清单列表。写入方通过发布新的元数据版本原子提交，读取方据此解析出一致快照。这样就能在不复制整个数据集的情况下获得 ACID 行为、模式强制、分区演进、行级删除和时间旅行。

目录提供名称、位置和访问控制。查询引擎读取元数据，按分区和统计信息裁剪文件，再只扫描需要的列。后台任务合并小文件并重写清单，避免摄入持续进行时查询规划性能下降。

何时使用

当一份数据必须同时服务 SQL 分析、流式摄入和机器学习，并且开放格式或避免供应商锁定很重要时，使用湖仓一体。它是云对象存储上新建分析平台的强默认选项。事务性写入仍应保留独立的事务数据库；湖仓不是低延迟 OLTP 存储的替代品。

权衡

湖仓减少重复和存储成本，却把复杂度转移到表维护：压缩、快照过期、文件大小和目录一致性。对于小型、高并发查询，延迟可能高于调优过的仓库。治理不会自动出现；没有归属和契约，湖仓会变成一个充满不可信表的快速数据湖。

示例与误解

一个流任务每分钟把事件写入按日期分区的 Iceberg 表。分析人员查询最新快照，夜间任务压缩文件并使旧快照过期。由于提交是原子的，读取方永远不会看到写了一半的批次。

常见误解是只要使用开放文件格式就是湖仓。没有表格式、目录和维护纪律，那仍然只是数据湖。格式提供机制；建模、质量检查和归属才提供信任。

代码示例

export interface CountMinOptions {
  width?: number;
  depth?: number;
  seeds?: readonly number[];
}

export class CountMinSketch {
  private readonly width: number;
  private readonly depth: number;
  private readonly table: Uint32Array;
  private readonly seeds: readonly number[];
  private total = 0;

  constructor(options: CountMinOptions = {}) {
    const width = options.width ?? 2048;
    const depth = options.depth ?? 5;
    if (!Number.isInteger(width) || width <= 0 || !Number.isInteger(depth) || depth <= 0) {
      throw new RangeError("width and depth must be positive integers");
    }
    this.width = width;
    this.depth = depth;
    this.table = new Uint32Array(width * depth);
    this.seeds = options.seeds ?? Array.from(
      { length: depth },
      (_, index) => Math.imul(index + 1, 0x9e3779b1) >>> 0,
    );
    if (this.seeds.length !== depth) {
      throw new RangeError("seeds length must equal depth");
    }
  }

  add(value: string, count = 1): this {
    if (!Number.isInteger(count) || count < 1) {
      throw new RangeError("count must be a positive integer");
    }
    for (let row = 0; row < this.depth; row += 1) {
      const index = row * this.width + this.hash(value, this.seeds[row]!);
      this.table[index] = (this.table[index]! + count) >>> 0;
    }
    this.total += count;
    return this;
  }

  estimate(value: string): number {
    let minimum = Number.POSITIVE_INFINITY;
    for (let row = 0; row < this.depth; row += 1) {
      minimum = Math.min(
        minimum,
        this.table[row * this.width + this.hash(value, this.seeds[row]!)]!,
      );
    }
    return minimum;
  }

  private hash(value: string, seed: number): number {
    let hash = seed >>> 0;
    for (let index = 0; index < value.length; index += 1) {
      hash ^= value.charCodeAt(index);
      hash = Math.imul(hash, 0x01000193) >>> 0;
    }
    return hash % this.width;
  }

  get count(): number {
    return this.total;
  }
}

export class SegmentTree {
  private readonly size: number;
  private readonly tree: number[];

  constructor(values: readonly number[]) {
    if (!Array.isArray(values)) throw new TypeError("values must be an array");
    for (const value of values) {
      if (typeof value !== "number" || !Number.isFinite(value)) {
        throw new TypeError("values must contain finite numbers");
      }
    }
    this.size = values.length;
    this.tree = new Array<number>(Math.max(4, this.size * 4)).fill(0);
    if (this.size > 0) this.build(values, 1, 0, this.size - 1);
  }

  sum(left: number, right: number): number {
    if (!Number.isInteger(left) || !Number.isInteger(right)) {
      throw new TypeError("range bounds must be integers");
    }
    if (left < 0 || right >= this.size || left > right) throw new RangeError("invalid range");
    return this.query(1, 0, this.size - 1, left, right);
  }

  update(index: number, value: number): void {
    if (!Number.isInteger(index) || index < 0 || index >= this.size) {
      throw new RangeError("index out of range");
    }
    if (typeof value !== "number" || !Number.isFinite(value)) {
      throw new TypeError("value must be a finite number");
    }
    this.assign(1, 0, this.size - 1, index, value);
  }

  private build(values: readonly number[], node: number, start: number, end: number): void {
    if (start === end) {
      this.tree[node] = values[start] ?? 0;
      return;
    }
    const middle = (start + end) >> 1;
    this.build(values, node * 2, start, middle);
    this.build(values, node * 2 + 1, middle + 1, end);
    this.tree[node] = this.tree[node * 2] + this.tree[node * 2 + 1];
  }

  private query(node: number, start: number, end: number, left: number, right: number): number {
    if (left <= start && end <= right) return this.tree[node];
    const middle = (start + end) >> 1;
    let total = 0;
    if (left <= middle) total += this.query(node * 2, start, middle, left, right);
    if (right > middle) total += this.query(node * 2 + 1, middle + 1, end, left, right);
    return total;
  }

  private assign(node: number, start: number, end: number, index: number, value: number): void {
    if (start === end) {
      this.tree[node] = value;
      return;
    }
    const middle = (start + end) >> 1;
    if (index <= middle) this.assign(node * 2, start, middle, index, value);
    else this.assign(node * 2 + 1, middle + 1, end, index, value);
    this.tree[node] = this.tree[node * 2] + this.tree[node * 2 + 1];
  }
}

const tree = new SegmentTree([2, 4, 6, 8, 10]);
console.log(tree.sum(1, 3));
tree.update(2, 0);
console.log(tree.sum(0, 4));
console.log(tree.sum(2, 2));