Integrate tqdm Progress Bar with aiohttp Requests in Python

In this tutorial, you’ll learn how to integrate the tqdm progress bar with aiohttp requests in Python.

This combination allows you to visualize the progress of your HTTP requests.

 

 

Basic Integration

To create a progress bar for a single aiohttp request, use the following code:

import asyncio
import aiohttp
from tqdm import tqdm
async def download_with_progress(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            total_size = int(response.headers.get('content-length', 0))
            with tqdm(total=total_size, unit='iB', unit_scale=True) as progress_bar:
                data = b''
                async for chunk in response.content.iter_chunked(102400):
                    size = len(chunk)
                    data += chunk
                    progress_bar.update(size)
    return data
url = 'http://localhost/myfile.zip'
asyncio.run(download_with_progress(url))

Output:

100%|██████████| 110M/110M [00:19<00:00, 5.50MiB/s]

This code creates a progress bar that updates as chunks of data are downloaded.

The bar shows the percentage complete, amount downloaded, total size, time elapsed, and download speed.

Handle unknown content length

To handle cases where content length is not provided, you can get file size from the header content-length:

import asyncio
import aiohttp
from tqdm import tqdm
async def download_with_progress(url):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            total_size = int(response.headers.get('content-length', 0))
            with tqdm(total=total_size, unit='iB', unit_scale=True, desc=url) as progress_bar:
                data = b''
                async for chunk in response.content.iter_chunked(102400):
                    size = len(chunk)
                    data += chunk
                    progress_bar.update(size)
                if total_size == 0:
                    progress_bar.total = progress_bar.n
    return data
url = 'http://localhost/unknown_size_file.bin'
asyncio.run(download_with_progress(url))

Output:

http://localhost/unknown_size_file.bin: 100%|██████████| 50.2M/50.2M [00:03<00:00, 1.73MiB/s]

 

Handle Multiple Requests

To use tqdm with multiple concurrent requests, you can use asyncio.gather() with tqdm for parallel requests

import asyncio
import aiohttp
from tqdm.asyncio import tqdm
async def download_url(session, url):
    async with session.get(url) as response:
        total_size = int(response.headers.get('content-length', 0))
        with tqdm(total=total_size, unit='iB', unit_scale=True, desc=url) as progress_bar:
            data = b''
            async for chunk in response.content.iter_chunked(102400):
                size = len(chunk)
                data += chunk
                progress_bar.update(size)
    return data
async def download_multiple(urls):
    async with aiohttp.ClientSession() as session:
        tasks = [download_url(session, url) for url in urls]
        await asyncio.gather(*tasks)
urls = ['http://localhost/file1.zip', 'http://localhost/file2.zip', 'http://localhost/file3.zip']
asyncio.run(download_multiple(urls))

Output:

http://localhost/file1.zip: 100%|██████████| 110M/110M [01:01<00:00, 1.79MiB/s]
http://localhost/file2.zip: 100%|█████████▉| 110M/110M [01:01<00:00, 691kiB/s]
http://localhost/file3.zip: 100%|██████████| 110M/110M [01:01<00:00, 687kiB/s]

This code displays individual progress bars for each file being downloaded concurrently.

 

Monitor Streaming Response Progress

For streaming responses, use the content.read() method and update the progress bar as the chunks are retrieved:

import asyncio
import aiohttp
from tqdm import tqdm
async def stream_download(url, chunk_size=8192):
    async with aiohttp.ClientSession() as session:
        async with session.get(url) as response:
            total_size = int(response.headers.get('content-length', 0))
            with tqdm(total=total_size, unit='iB', unit_scale=True, desc=url) as progress_bar:
                data = b''
                while True:
                    chunk = await response.content.read(chunk_size)
                    if not chunk:
                        break
                    data += chunk
                    progress_bar.update(len(chunk))
    return data
url = 'http://localhost/large_stream.mp4'
asyncio.run(stream_download(url))

Output:

http://localhost/large_stream.mp4: 100%|██████████| 153M/153M [00:44<00:00, 3.42MiB/s]

This code streams the response content and updates the progress bar as chunks are received.

 

Retry with Progress

To implement retry attempts with a progress bar, you can reset the progress bar for each attempt and show which attempt is currently in progress:

import asyncio
import aiohttp
from tqdm import tqdm
async def download_with_reset_on_retry(url, max_retries=3):
    async with aiohttp.ClientSession() as session:
        for attempt in range(max_retries):
            try:
                async with session.get(url) as response:
                    total_size = int(response.headers.get('content-length', 0))
                    with tqdm(total=total_size, unit='iB', unit_scale=True, 
                              desc=f"{url} (Attempt {attempt+1}/{max_retries})") as progress_bar:
                        data = b''
                        async for chunk in response.content.iter_chunked(1024):
                            size = len(chunk)
                            data += chunk
                            progress_bar.update(size)
                    return data
            except aiohttp.ClientError:
                if attempt == max_retries - 1:
                    tqdm.write(f"Failed to download {url} after {max_retries} attempts")
                    raise
                await asyncio.sleep(2 ** attempt)
url = 'http://localhost/unstable_large_file.zip'
asyncio.run(download_with_reset_on_retry(url))

Output:

http://localhost/unstable_large_file.zip (Attempt 1/3):  6%|█████     | 6.56M/110.0M [00:04<02:01, 848kiB/s]
http://localhost/unstable_large_file.zip (Attempt 2/3): 100%|██████████| 110.0M/110.0M [00:04<00:00, 598kiB/s]

 

Upload Files with Progress Bars

To upload files with progress tracking, you can use multipart requests and show progress for each file upload:

import asyncio
import aiohttp
from tqdm import tqdm
import os
async def upload_file(session, url, file_path):
    file_size = os.path.getsize(file_path)
    with tqdm(total=file_size, unit='iB', unit_scale=True, desc=os.path.basename(file_path)) as progress_bar:
        with open(file_path, 'rb') as f:
            async with session.post(url, data=ProgressReader(f, progress_bar)) as response:
                return await response.text()
class ProgressReader:
    def __init__(self, file_obj, progress_bar):
        self.file_obj = file_obj
        self.progress_bar = progress_bar
    async def read(self, size):
        data = self.file_obj.read(size)
        self.progress_bar.update(len(data))
        return data
async def upload_files(url, file_paths):
    async with aiohttp.ClientSession() as session:
        tasks = [upload_file(session, url, file_path) for file_path in file_paths]
        results = await asyncio.gather(*tasks)
    return results
url = 'http://localhost/upload'
file_paths = ['large_file1.zip', 'large_file2.pdf', 'large_file3.mp4']
asyncio.run(upload_files(url, file_paths))

Output:

large_file1.zip: 100%|██████████| 50.0M/50.0M [00:25<00:00, 2.00MiB/s]
large_file2.pdf: 100%|██████████| 25.0M/25.0M [00:12<00:00, 2.08MiB/s]
large_file3.mp4: 100%|██████████| 100M/100M [00:50<00:00, 2.00MiB/s]