python asyncio从3.4迁移到3.5+

Question

大家晚上好,我正在尝试创建互联网机器人,我在将我的脚本从python 3.4迁移到3.5或3.6+时遇到了问题.它使用asyncio并且在3.4 python上运行良好但是当我用python3.5 +启动时我得到了错误:RuntimeError: Cannot run the event loop while another loop is running

这是代码方案:

import multiprocessing as mp
import asyncio
import concurrent.futures
import aiohttp

def create_proccesses(separate_loop_creator, coro):
    proccesses = []
    for n in range(2):
        proc = mp.Process(target=separate_loop_creator, args=(coro,))
        proc.start()
        proccesses.append(proc)
    for p in proccesses:
        p.join()

def separate_loop_creator(coro):
    sep_loop = asyncio.new_event_loop()
    asyncio.set_event_loop(sep_loop)
    tasks = [asyncio.async(coro(sep_loop)) for _ in range(100)]
    try:
        sep_loop.run_until_complete(asyncio.wait(tasks))
        sep_loop.close()
    except Exception as err:
        print(err)
        for task in tasks:
            task.cancel()
        sep_loop.close()


@asyncio.coroutine
def manager(exe, loop):
    # some calculations and start coros in several processes
    loop.run_in_executor(
        exe,
        create_proccesses,
        separate_loop_creator,
        some_coro
    )

@asyncio.coroutine
def some_work_in_mainloop():
    while True:
        print('Some server dealing with connections here...')
        yield from asyncio.sleep(1)

@asyncio.coroutine
def some_coro(loop):
    with aiohttp.ClientSession(loop=loop) as session:
        response = yield from session.get('http://google.com')
        yield from asyncio.sleep(2)
        print(response.status)

if __name__ == '__main__':
    mainloop = asyncio.get_event_loop()
    executor = concurrent.futures.ProcessPoolExecutor(5)
    asyncio.async(some_work_in_mainloop())
    asyncio.async(manager(executor, mainloop))
    try:
        mainloop.run_forever()
    finally:
        mainloop.close()

separate_loop_creator()协程中的异常提升,它是RuntimeError: Cannot run the event loop while another loop is running.我认为这是因为改变了机制get_event_loop(),但我不明白我的代码有什么问题.

这是我想要做的:

                       +--------------+
               +-------+other service |
    +----------+       +--------------+
    | mainloop |
    +----------+
               |     +------------+
               +-----+   executor |
                     +------+-----+
                            |
                     +------+--------+
                     |start proccess |
                     +---+-------+---+
+-----------------+      |       |      +---------------+
|start new loop   +------+       +------+ start new loop|
+--------+--------+                     +-------+-------+
         |                                      |
 +-------+-------+                       +------v-------+
 |   run coro    |                       | run coro     |
 +---------------+                       +--------------+

这是我在python3.5.3上得到的跟踪:

Traceback (most recent call last):
  File "tst.py", line 21, in separate_loop_creator
    sep_loop.run_until_complete(asyncio.wait(tasks))
  File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 454, in run_until_complete
    self.run_forever()
  File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 411, in run_forever
    'Cannot run the event loop while another loop is running')
RuntimeError: Cannot run the event loop while another loop is running
Cannot run the event loop while another loop is running
Traceback (most recent call last):
  File "tst.py", line 21, in separate_loop_creator
    sep_loop.run_until_complete(asyncio.wait(tasks))
  File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 454, in run_until_complete
    self.run_forever()
  File "/root/.pyenv/versions/3.5.3/lib/python3.5/asyncio/base_events.py", line 411, in run_forever
    'Cannot run the event loop while another loop is running')
RuntimeError: Cannot run the event loop while another loop is running

Python 3.4.3结果:

...
200
Some server dealing with connections here...
200
200
Some server dealing with connections here...
200
200
Some server dealing with connections here...
200
...

Answer 1

这实际上是CPython 3.6.0中asyncio的一个错误.有一个公关来解决这个问题,所以3.6.1按预期工作.

作为一种解决方法,您可以在项目中添加以下代码:

import sys

if sys.version_info[:3] == (3, 6, 0):
    import asyncio.events as _ae
    import os as _os

    _ae._RunningLoop._pid = None

    def _get_running_loop():
        if _ae._running_loop._pid == _os.getpid():
            return _ae._running_loop._loop

    def _set_running_loop(loop):
        _ae._running_loop._pid = _os.getpid()
        _ae._running_loop._loop = loop

    _ae._get_running_loop = _get_running_loop
    _ae._set_running_loop = _set_running_loop