Go NDJSON Streaming with json.Decoder

Decode newline-delimited JSON events one at a time with json.Decoder, validate each record, and aggregate without buffering the file.

GoData Fetching

Code

snippet
package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"strings"
)

type Event struct {
	ID          int               `json:"id"`
	Type        string            `json:"type"`
	AmountCents int64             `json:"amount_cents"`
	Meta        map[string]string `json:"meta,omitempty"`
}

func streamNDJSON(r io.Reader, handle func(Event) error) error {
	decoder := json.NewDecoder(r)
	for {
		var event Event
		if err := decoder.Decode(&event); err != nil {
			if errors.Is(err, io.EOF) {
				return nil
			}
			return fmt.Errorf("decode event: %w", err)
		}
		if event.ID <= 0 {
			return fmt.Errorf("event %d: id must be positive", event.ID)
		}
		if event.Type == "" {
			return fmt.Errorf("event %d: type is required", event.ID)
		}
		if event.AmountCents < 0 {
			return fmt.Errorf("event %d: amount_cents must be >= 0", event.ID)
		}
		if err := handle(event); err != nil {
			return err
		}
	}
}

func main() {
	payload := strings.Join([]string{
		`{"id":1,"type":"credit","amount_cents":1250}`,
		`{"id":2,"type":"debit","amount_cents":325,"meta":{"channel":"api"}}`,
		`{"id":3,"type":"credit","amount_cents":900}`,
	}, "\n")

	var netCents int64
	counts := map[string]int{}
	err := streamNDJSON(strings.NewReader(payload), func(event Event) error {
		counts[event.Type]++
		if event.Type == "credit" {
			netCents += event.AmountCents
		} else {
			netCents -= event.AmountCents
		}
		return nil
	})
	fmt.Printf("counts=%v net_cents=%d err=%v\n", counts, netCents, err)

	bad := bytes.NewBufferString("{\"id\":4,\"type\":\"credit\",\"amount_cents\":500}\n{\"id\":5,\"amount_cents\":100}\n")
	seen := 0
	err = streamNDJSON(bad, func(Event) error {
		seen++
		return nil
	})
	fmt.Printf("seen=%d err=%v\n", seen, err)
}