中文
devexamples

AIOHTTP Streaming Chunked Download

Download large files with aiohttp while iterating response chunks, hashing each block on the fly and keeping memory usage flat.

PythonData Fetching

Code

snippet
# stream chunks incrementally to disk instead of buffering
import asyncio
import hashlib

import aiohttp


async def download_and_hash(url: str, block_hint: int = 1024 * 64) -> dict:
    digest = hashlib.sha256()
    received = 0
    chunks = 0

    timeout = aiohttp.ClientTimeout(total=None, sock_read=30)
    async with aiohttp.ClientSession(timeout=timeout) as session:
        async with session.get(url) as resp:
            resp.raise_for_status()
            total = int(resp.headers.get('Content-Length', 0))

            async for chunk in resp.content.iter_chunked(block_hint):
                digest.update(chunk)
                received += len(chunk)
                chunks += 1
                if chunks % 200 == 0:
                    print(f'progress {received / total:.1%}')

    return {
        'bytes': received,
        'chunks': chunks,
        'sha256': digest.hexdigest(),
    }


result = asyncio.run(
    download_and_hash('https://cdn.example.com/datasets/prices.parquet')
)
print('verified sha256:', result['sha256'][:16], 'bytes:', result['bytes'])

Related Examples