中文
devexamples

Pandas Chunked CSV Aggregation

Aggregate CSV files larger than memory with pandas chunksize iteration, accumulating partial sums and merging into a final report.

PythonData Integrity

Code

snippet
import pandas as pd

CHUNK_ROWS = 100_000


def aggregate_csv(path: str) -> pd.DataFrame:
    partials: list[pd.DataFrame] = []
    total_rows = 0

    reader = pd.read_csv(
        path,
        chunksize=CHUNK_ROWS,
        usecols=['region', 'category', 'amount', 'quantity'],
        dtype={'region': 'category', 'category': 'category'},
    )

    for chunk in reader:
        total_rows += len(chunk)
        partial = (
            chunk
            .groupby(['region', 'category'], observed=True)
            .agg(revenue=('amount', 'sum'), units=('quantity', 'sum'))
            .reset_index()
        )
        partials.append(partial)

    combined = pd.concat(partials, ignore_index=True)
    final = (
        combined
        .groupby(['region', 'category'], observed=True)
        .agg(revenue=('revenue', 'sum'), units=('units', 'sum'))
        .reset_index()
        .sort_values('revenue', ascending=False)
    )
    final.attrs['source_rows'] = total_rows
    return final


report = aggregate_csv('sales/2026-10-transactions.csv')
print('regions reported:', len(report))
print('source rows:', report.attrs['source_rows'])

Related Examples