什么是故障注入
故障注入故意引入特定失败：依赖返回错误，调用耗时三秒而不是三十毫秒，磁盘写满，进程被杀，网络丢包。团队随后观察系统能否检测、隔离并在承诺时间内恢复。与等待真实故障不同，注入让失败可复现、可归因且可以安全研究。
故障类型
有用的故障包括延迟、错误响应、连接重置、资源耗尽、时钟偏移和部分数据丢失。延迟往往最能揭示问题，因为它会触发超时、重试和队列增长，而纯错误注入不会。组合故障同样重要：慢响应加重试风暴可能耗尽连接池，而任一单独故障都能承受。故障语义要匹配真实依赖行为，而不是发明方便的失败。
层级
注入可以在测试代码中通过抛出或延迟的包装器完成，也可以在网络层通过代理或服务网格完成，还可以在基础设施层终止实例或阻断区域。层级越低越便宜精确，层级越高覆盖的真实系统越多。先从低层开始，再逐步升级。
安全与局限
开始前要定义假设、稳态指标、受控的爆炸半径和终止条件。未经负责人同意，不要向共享依赖注入故障。生产环境应从一小部分流量和非高峰时段开始。记录实验、观察和后续行动。故障注入是一种技术，混沌工程是运行实验以验证韧性的更广泛学科。一次通过的注入只证明某一种故障在某一配置下被处理。例如，杀掉一个 Pod 可能成功，而整个区域丢失仍会失败。应把结果用于修正超时预算和故障转移路径，而不是宣布韧性已经完成。
代码示例
package main
import (
"errors"
"fmt"
"sync"
"time"
)
type State int
const (
StateClosed State = iota
StateOpen
StateHalfOpen
)
func (s State) String() string {
switch s {
case StateClosed:
return "closed"
case StateOpen:
return "open"
case StateHalfOpen:
return "half-open"
default:
return "unknown"
}
}
var ErrOpen = errors.New("circuit breaker is open")
type Breaker struct {
mu sync.Mutex
threshold int
cooldown time.Duration
now func() time.Time
state State
failures int
openedAt time.Time
}
func NewBreaker(threshold int, cooldown time.Duration) (*Breaker, error) {
if threshold < 1 {
return nil, fmt.Errorf("threshold must be positive")
}
if cooldown <= 0 {
return nil, fmt.Errorf("cooldown must be positive")
}
return &Breaker{threshold: threshold, cooldown: cooldown, now: time.Now}, nil
}
func (b *Breaker) Allow() error {
b.mu.Lock()
defer b.mu.Unlock()
if b.state == StateOpen {
if b.now().Sub(b.openedAt) < b.cooldown {
return ErrOpen
}
b.state = StateHalfOpen
}
return nil
}
func (b *Breaker) record(err error) {
b.mu.Lock()
defer b.mu.Unlock()
if err == nil {
b.state = StateClosed
b.failures = 0
return
}
b.failures++
if b.state == StateHalfOpen || b.failures >= b.threshold {
b.state = StateOpen
b.openedAt = b.now()
}
}
func (b *Breaker) Execute(operation func() error) error {
if err := b.Allow(); err != nil {
return err
}
err := operation()
b.record(err)
return err
}
func (b *Breaker) CurrentState() State {
b.mu.Lock()
defer b.mu.Unlock()
return b.state
}
type fakeClock struct {
current time.Time
}
func (c *fakeClock) Now() time.Time { return c.current }
func (c *fakeClock) Advance(step time.Duration) { c.current = c.current.Add(step) }
func main() {
clock := &fakeClock{current: time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)}
breaker, err := NewBreaker(2, 100*time.Millisecond)
if err != nil {
fmt.Println(err)
return
}
breaker.now = clock.Now
failing := func() error { return errors.New("upstream unavailable") }
for call := 1; call <= 3; call++ {
fmt.Printf(
"call %d: err=%v state=%s\n",
call,
breaker.Execute(failing),
breaker.CurrentState(),
)
}
clock.Advance(150 * time.Millisecond)
fmt.Println("after cooldown state:", breaker.CurrentState())
probeErr := breaker.Execute(func() error { return nil })
fmt.Printf("probe err=%v state=%s\n", probeErr, breaker.CurrentState())
}const RETRYABLE = new Set([408, 425, 429, 500, 502, 503, 504]);
const CACHEABLE = new Set([200, 203, 204, 206, 300, 301, 308, 404, 410]);
export function classifyStatus(status) {
if (!Number.isInteger(status) || status < 100 || status > 599) {
throw new RangeError("status must be a three-digit HTTP status code");
}
const group = Math.floor(status / 100);
const labels = {
1: "informational",
2: "success",
3: "redirect",
4: "client-error",
5: "server-error",
};
return {
status,
group,
label: labels[group],
success: group === 2,
redirect: group === 3,
error: group >= 4,
retryable: RETRYABLE.has(status),
cacheable: CACHEABLE.has(status),
};
}
export function isRetryableStatus(status) {
return classifyStatus(status).retryable;
}