import asyncio
import aiohttp
import socket
def _create_loop():
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop = asyncio.get_event_loop()
return loop
async def _create_tasks(loop, URLs, func):
connector = aiohttp.TCPConnector(limit=200,
limit_per_host=200,
force_close=True,
enable_cleanup_closed=True,
family=socket.AF_INET, verify_ssl=False)
async with aiohttp.ClientSession(loop=loop, connector=connector) as session:
semaphore = asyncio.Semaphore(200)
async with semaphore:
tasks = [asyncio.create_task(func(session, URL)) for URL in URLs]
return await asyncio.gather(*tasks)
async def _fetch_data_async(session, url):
async with session.get(url) as response:
return await response.json()
loop = _create_loop()
tasks = _create_tasks(loop, URL_ls, _fetch_data_async)
results = loop.run_until_complete(tasks)
loop.close()
Run Code Online (Sandbox Code Playgroud)
我的 API 提供商限制每个请求 …