Go NDJSON Streaming with json.Decoder
Decode newline-delimited JSON events one at a time with json.Decoder, validate each record, and aggregate without buffering the file.
Code
package main
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"strings"
)
type Event struct {
ID int `json:"id"`
Type string `json:"type"`
AmountCents int64 `json:"amount_cents"`
Meta map[string]string `json:"meta,omitempty"`
}
func streamNDJSON(r io.Reader, handle func(Event) error) error {
decoder := json.NewDecoder(r)
for {
var event Event
if err := decoder.Decode(&event); err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return fmt.Errorf("decode event: %w", err)
}
if event.ID <= 0 {
return fmt.Errorf("event %d: id must be positive", event.ID)
}
if event.Type == "" {
return fmt.Errorf("event %d: type is required", event.ID)
}
if event.AmountCents < 0 {
return fmt.Errorf("event %d: amount_cents must be >= 0", event.ID)
}
if err := handle(event); err != nil {
return err
}
}
}
func main() {
payload := strings.Join([]string{
`{"id":1,"type":"credit","amount_cents":1250}`,
`{"id":2,"type":"debit","amount_cents":325,"meta":{"channel":"api"}}`,
`{"id":3,"type":"credit","amount_cents":900}`,
}, "\n")
var netCents int64
counts := map[string]int{}
err := streamNDJSON(strings.NewReader(payload), func(event Event) error {
counts[event.Type]++
if event.Type == "credit" {
netCents += event.AmountCents
} else {
netCents -= event.AmountCents
}
return nil
})
fmt.Printf("counts=%v net_cents=%d err=%v\n", counts, netCents, err)
bad := bytes.NewBufferString("{\"id\":4,\"type\":\"credit\",\"amount_cents\":500}\n{\"id\":5,\"amount_cents\":100}\n")
seen := 0
err = streamNDJSON(bad, func(Event) error {
seen++
return nil
})
fmt.Printf("seen=%d err=%v\n", seen, err)
}Report a problem with this exampleOpens a pre-filled issue on GitHub.