A/B 测试统计显著性

概念定义

随机化造就两组只在处理上不同的总体；噪声也会让它们不同。显著性机器量化“在无效应假设下观察到这个差距有多意外”，让决策建立在证据而不是热情上。

工作机制

预注册：主指标、方向、最小可检效应、显著性水平。功效分析给出样本量与时长——覆盖完整周循环以吸收星期效应。到期用固定分析评估；扫描指标族时施加 Bonferroni 或错误发现率校正；发布任何赢之前先查护栏无伤害。

使用场景

一切以数据为依据的决策：UI 变更、算法调优、带用户可见风险的基础设施成本变更。把检查单编码进实验平台，违规需要显式越权。

常见误区

p<0.05 不等于“功能有效”——按构造每二十次就有一次纯噪声达标；每天偷看、一显著就停的“可选停止”会大幅抬高假阳性——确需早看要用序贯方法。指标越多误报族越大，不校正就是挑显著值报告。样本比例不匹配暴露的是埋点 bug，直接作废整轮实验；新奇效应几周后消退，要留 holdout 队列测耐久；超巨样本下的微小效应统计真实但实践无意义——报告效应量与区间，而不只是显著性。

代码示例

export type Outcome<T> = { readonly ok: true; readonly value: T } | { readonly ok: false; readonly error: Error };

export function ok<T>(value: T): Outcome<T> {
  return { ok: true, value };
}

export function fail(message: string): Outcome<never> {
  return { ok: false, error: new Error(message) };
}

export interface AttemptLog {
  readonly index: number;
  readonly at: number;
  readonly result: "ok" | "error";
}

export interface RetryHarness<T> {
  readonly attempts: AttemptLog[];
  readonly loader: () => Promise<T>;
  readonly exhausted: () => boolean;
}

export function createHarness<T>(
  script: readonly Outcome<T>[],
  clock: { now: () => number; setTimeout: (fn: () => void, ms: number) => unknown },
): RetryHarness<T> {
  const attempts: AttemptLog[] = [];
  let index = 0;

  const loader = async (): Promise<T> =>
    new Promise<T>((resolve, reject) => {
      const outcome = script[index];
      attempts.push({
        index,
        at: clock.now(),
        result: outcome.ok ? "ok" : "error",
      });
      index += 1;
      if (outcome.ok) resolve(outcome.value);
      else reject(outcome.error);
    });

  return {
    attempts,
    loader,
    exhausted: () => index >= script.length,
  };
}

package main

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

func WriteGolden(target string, data []byte) error {
	if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
		return fmt.Errorf("create golden dir: %w", err)
	}
	return os.WriteFile(target, data, 0o644)
}

func diffLines(want, got string) string {
	wantLines := strings.Split(strings.TrimSpace(want), "\n")
	gotLines := strings.Split(strings.TrimSpace(got), "\n")
	limit := len(wantLines)
	if len(gotLines) < limit {
		limit = len(gotLines)
	}
	var builder strings.Builder
	for i := 0; i < limit; i++ {
		if wantLines[i] != gotLines[i] {
			fmt.Fprintf(&builder, "- %s\n+ %s\n", wantLines[i], gotLines[i])
		}
	}
	if len(wantLines) != len(gotLines) {
		fmt.Fprintf(&builder, "line count: want %d got %d\n", len(wantLines), len(gotLines))
	}
	return builder.String()
}

func CompareGolden(target string, got []byte) error {
	want, err := os.ReadFile(target)
	if err != nil {
		return fmt.Errorf("read golden: %w", err)
	}
	if !bytes.Equal(bytes.TrimSpace(got), bytes.TrimSpace(want)) {
		return fmt.Errorf("golden mismatch for %s\n%s", filepath.Base(target), diffLines(string(want), string(got)))
	}
	return nil
}

func main() {
	dir, err := os.MkdirTemp("", "golden-*")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(dir)

	target := filepath.Join(dir, "report.golden")
	report := []byte("users=3\norders=5\n")
	if err := WriteGolden(target, report); err != nil {
		panic(err)
	}
	if err := CompareGolden(target, report); err != nil {
		fmt.Println("unexpected mismatch:", err)
	} else {
		fmt.Println("match: ok")
	}
	if err := CompareGolden(target, []byte("users=4\norders=5\n")); err != nil {
		fmt.Println("mismatch detected:")
		fmt.Println(err)
	}
}