Multi Stage Pipeline With Fail Fast Ordering
A staged data pipeline that runs named stages in order, stops at the first failure, and reports which stage and record index broke.
Code
from typing import Any, Callable, List, Optional, Sequence, Tuple
class StageError(RuntimeError):
"""Raised when a pipeline stage fails or breaks the record invariant."""
def __init__(self, stage: str, index: Optional[int], message: str) -> None:
self.stage = stage
self.index = index
location = stage if index is None else "%s[%d]" % (stage, index)
super().__init__("%s: %s" % (location, message))
def run_pipeline(
records: Sequence[Any],
stages: Sequence[Tuple[str, Callable[[Any], Any]]],
) -> List[Any]:
"""Run named stages in order; fail fast with stage and index context."""
if not records:
raise StageError("pipeline", None, "no input records")
current: List[Any] = list(records)
for name, stage in stages:
produced: List[Any] = []
for index, record in enumerate(current):
try:
produced.append(stage(record))
except Exception as exc:
raise StageError(name, index, str(exc)) from exc
if len(produced) != len(current):
raise StageError(
name,
None,
"record count changed from %d to %d" % (len(current), len(produced)),
)
current = produced
return currentReport a problem with this exampleOpens a pre-filled issue on GitHub.