测试预言问题

概念概述

测试预言问题（The Test Oracle Problem）指的是：测试预言问题是如何确定正确的预期输出，它无法从被测代码推导，往往是测试中最难的部分。

从更严格的定义看，The oracle problem is the difficulty of determining, for a given input, what the correct expected result of a program should be, independent of the implementation under test. 该概念在工程实践中的价值，在于它能帮助团队用统一的语言描述一个具体的质量属性或设计约束，从而在评审与协作中减少歧义。

为什么需要它

It surfaces wherever expected values are expensive to compute, ambiguous, or only knowable by running the very code being tested. 它与 Branch Coverage、Stress Testing Beyond Design Limits、Load Testing to Target Concurrency 等概念经常一起出现，用于解决相近的工程问题。理解它的适用场景，有助于在多种备选方案之间做出有依据的取舍，而不是凭直觉选择技术方案。

常见误解

• the oracle is always the specification and is therefore trivial
• comparing against the same implementation is a valid oracle

需要避免的反模式

• using the code under test to generate its own expected values
• assuming a green suite means the requirements are correct

实践要点

1. Techniques such as metamorphic testing relate inputs and outputs, for example asserting that sorting a list preserves its multiset, so correctness can be checked without a fixed expected value.
2. Differential testing compares two independent implementations against the same input, which is powerful when a reference exists but silent if both share a bug.
3. When no trustworthy oracle exists, properties, invariants, and human review substitute for exact expected values.

参考来源

https://en.wikipedia.org/wiki/Test_oracle

代码示例

package main

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

func WriteGolden(target string, data []byte) error {
	if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
		return fmt.Errorf("create golden dir: %w", err)
	}
	return os.WriteFile(target, data, 0o644)
}

func diffLines(want, got string) string {
	wantLines := strings.Split(strings.TrimSpace(want), "\n")
	gotLines := strings.Split(strings.TrimSpace(got), "\n")
	limit := len(wantLines)
	if len(gotLines) < limit {
		limit = len(gotLines)
	}
	var builder strings.Builder
	for i := 0; i < limit; i++ {
		if wantLines[i] != gotLines[i] {
			fmt.Fprintf(&builder, "- %s\n+ %s\n", wantLines[i], gotLines[i])
		}
	}
	if len(wantLines) != len(gotLines) {
		fmt.Fprintf(&builder, "line count: want %d got %d\n", len(wantLines), len(gotLines))
	}
	return builder.String()
}

func CompareGolden(target string, got []byte) error {
	want, err := os.ReadFile(target)
	if err != nil {
		return fmt.Errorf("read golden: %w", err)
	}
	if !bytes.Equal(bytes.TrimSpace(got), bytes.TrimSpace(want)) {
		return fmt.Errorf("golden mismatch for %s\n%s", filepath.Base(target), diffLines(string(want), string(got)))
	}
	return nil
}

func main() {
	dir, err := os.MkdirTemp("", "golden-*")
	if err != nil {
		panic(err)
	}
	defer os.RemoveAll(dir)

	target := filepath.Join(dir, "report.golden")
	report := []byte("users=3\norders=5\n")
	if err := WriteGolden(target, report); err != nil {
		panic(err)
	}
	if err := CompareGolden(target, report); err != nil {
		fmt.Println("unexpected mismatch:", err)
	} else {
		fmt.Println("match: ok")
	}
	if err := CompareGolden(target, []byte("users=4\norders=5\n")); err != nil {
		fmt.Println("mismatch detected:")
		fmt.Println(err)
	}
}

package calc

import "testing"

func Divide(a, b float64) (float64, bool) {
	if b == 0 {
		return 0, false
	}
	return a / b, true
}

func TestDivide(t *testing.T) {
	cases := []struct {
		name    string
		a, b    float64
		want    float64
		wantOK  bool
	}{
		{name: "even", a: 10, b: 2, want: 5, wantOK: true},
		{name: "fraction", a: 1, b: 4, want: 0.25, wantOK: true},
		{name: "divide by zero", a: 1, b: 0, want: 0, wantOK: false},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			got, ok := Divide(tc.a, tc.b)
			if ok != tc.wantOK || got != tc.want {
				t.Fatalf("got (%v, %v), want (%v, %v)", got, ok, tc.want, tc.wantOK)
			}
		})
	}
}