I think this will work, i need to test further but it seems to work on
python 3.8.5. I think the prev fix didnt really fix it. And the core of the
problem was " _done_callback" fired before "outer" was defined by "
_GatheringFuture" . Have a look at "Look here" below.
Let me know if you agree, or otherwise. Thanks
def gather(*coros_or_futures, loop=None, return_exceptions=False):
if not coros_or_futures:
if loop is None:
loop = events.get_event_loop()
else:
warnings.warn("The loop argument is deprecated since Python
3.8, "
"and scheduled for removal in Python 3.10.",
DeprecationWarning, stacklevel=2)
outer = loop.create_future()
outer.set_result([])
return outer
def _done_callback(fut):
while not outer: <==== Look here
print(f"scream ... you can't just return")
nonlocal nfinished
nfinished += 1
if outer.done():
if not fut.cancelled():
# Mark exception retrieved.
fut.exception()
return
if not return_exceptions:
if fut.cancelled():
# Check if 'fut' is cancelled first, as
# 'fut.exception()' will *raise* a CancelledError
# instead of returning it.
exc = exceptions.CancelledError()
outer.set_exception(exc)
return
else:
exc = fut.exception()
if exc is not None:
outer.set_exception(exc)
return
if nfinished == nfuts:
# All futures are done; create a list of results
# and set it to the 'outer' future.
results = []
for fut in children:
if fut.cancelled():
# Check if 'fut' is cancelled first, as
# 'fut.exception()' will *raise* a CancelledError
# instead of returning it.
res = exceptions.CancelledError()
else:
res = fut.exception()
if res is None:
res = fut.result()
results.append(res)
if outer._cancel_requested:
# If gather is being cancelled we must propagate the
# cancellation regardless of *return_exceptions* argument.
# See issue 32684.
outer.set_exception(exceptions.CancelledError())
else:
outer.set_result(results)
arg_to_fut = {}
children = []
nfuts = 0
nfinished = 0
outer = None <==== Look here, need at least define the variable
for arg in coros_or_futures:
if arg not in arg_to_fut:
fut = ensure_future(arg, loop=loop)
if loop is None:
loop = futures._get_loop(fut)
if fut is not arg:
# 'arg' was not a Future, therefore, 'fut' is a new
# Future created specifically for 'arg'. Since the caller
# can't control it, disable the "destroy pending task"
# warning.
fut._log_destroy_pending = False
nfuts += 1
arg_to_fut[arg] = fut
fut.add_done_callback(_done_callback) <==== Look here,
_done_callback fired before _GatheringFuture got a chance to return
"outer"!?
else:
# There's a duplicate Future object in coros_or_futures.
fut = arg_to_fut[arg]
children.append(fut)
outer = _GatheringFuture(children, loop=loop) <==== Look here,
"outer" defined only here.
return outer
On Sat, Feb 26, 2022 at 4:56 PM Norman Fung <report@bugs.python.org> wrote:
>
> Norman Fung <norman.lm.fung@gmail.com> added the comment:
>
> I tried hacking tasks.py (After revert back to Python 3.8.5), it
> didn't work: Error disappeared, but essentially the program execution
> freezed.
>
> def _done_callback(fut):
> ... more ...
>
> try: outer
> except NameError: outer = None
> if outer is None or outer.done():
> if not fut.cancelled():
> # Mark exception retrieved.
> fut.exception()
> return
> ... more code here ...
>
> arg_to_fut = {}
> children = []
> nfuts = 0
> nfinished = 0
> outer = None =============> added
> for arg in coros_or_futures:
> if arg not in arg_to_fut:
> fut = ensure_future(arg, loop=loop)
>
> Reference: https://github.com/python/cpython/pull/31441/files
>
> On Sat, Feb 26, 2022 at 3:44 PM Norman Fung <report@bugs.python.org>
> wrote:
>
> >
> > Norman Fung <norman.lm.fung@gmail.com> added the comment:
> >
> > Also, i reverted back to python 3.8.5 and overwrote task.py as
> recommended.
> > I think that version we dont already have "GenericAlias".
> >
> > import asyncio
> > File "C:\ProgramData\Anaconda3\lib\asyncio\__init__.py", line 8, in
> > <module>
> > from .base_events import *
> > File "C:\ProgramData\Anaconda3\lib\asyncio\base_events.py", line 45, in
> > <module>
> > from . import staggered
> > File "C:\ProgramData\Anaconda3\lib\asyncio\staggered.py", line 11, in
> > <module>
> > from . import tasks
> > File "C:\ProgramData\Anaconda3\lib\asyncio\tasks.py", line 20, in
> > <module>
> > from types import GenericAlias
> > ImportError: cannot import name 'GenericAlias' from 'types'
> > (C:\ProgramData\Anaconda3\lib\types.py)
> >
> > On Sat, Feb 26, 2022 at 3:20 PM Norman Fung <report@bugs.python.org>
> > wrote:
> >
> > >
> > > Norman Fung <norman.lm.fung@gmail.com> added the comment:
> > >
> > > Thanks Andrew for heads up.
> > >
> > > *1. My laptop (Windows 10) *with no changes runs happily with no error
> > from
> > > here.
> > > python 3.8.5
> > > asyncio 3.4.3
> > >
> > > 2. *My Windows VM (AWS EC2)* is where the whole mess is happening. I
> > > *upgraded
> > > *from Python 3.8.5 to 3.9.7. Asynio stayed 3.4.3, no change.
> > >
> > > I manually overwrite C:\ProgramData\Anaconda3\Lib\asyncio\task.py with
> > > what's here https://github.com/python/cpython/pull/31441/files
> > > (Only
> > >
> > >
> >
> https://github.com/asvetlov/cpython/blob/150ef068c77abc6a5e7ba97397ac65113dba355a/Lib/asyncio/tasks.py
> > > )
> > >
> > > Before I made this change, the error was:
> > > File "src\xxx\xxx.py", line 37, in _invoke_runners
> > > one_loop.run_until_complete(runner.xxx(xxx, xxx))
> > > File "C:\ProgramData\Anaconda3\lib\site-packages\nest_asyncio.py",
> line
> > > 90, in run_until_complete
> > > self._run_once()
> > > File "C:\ProgramData\Anaconda3\lib\site-packages\nest_asyncio.py",
> line
> > > 127, in _run_once
> > > handle._run()
> > > File "C:\ProgramData\Anaconda3\lib\site-packages\nest_asyncio.py",
> line
> > > 196, in run
> > > ctx.run(self._callback, *self._args)
> > > File "C:\ProgramData\Anaconda3\lib\asyncio\futures.py", line 356, in
> > > _set_state
> > > _copy_future_state(other, future)
> > > File "C:\ProgramData\Anaconda3\lib\asyncio\futures.py", line 335, in
> > > _copy_future_state
> > > dest.set_result(result)
> > > File "C:\ProgramData\Anaconda3\lib\asyncio\futures.py", line 237, in
> > > set_result
> > > self.__schedule_callbacks()
> > > File "C:\ProgramData\Anaconda3\lib\asyncio\futures.py", line 149, in
> > > __schedule_callbacks
> > > self._loop.call_soon(callback, self, context=ctx)
> > > Traceback (most recent call last):
> > > File "C:\ProgramData\Anaconda3\lib\site-packages\nest_asyncio.py",
> line
> > > 196, in run
> > > ctx.run(self._callback, *self._args)
> > > File "C:\ProgramData\Anaconda3\lib\asyncio\tasks.py", line 762, in
> > > _done_callback
> > > if outer.done():
> > > NameError: free variable 'outer' referenced before assignment in
> > enclosing
> > > scope
> > >
> > > After I upgraded python to 3.9.7 and overwrite task.py, a different
> error
> > > (but also in asyncio stack):
> > >
> > > 2022-02-26 06:48:27,047 casin0: @slack algo_order.id: 13 #8 Algo tick
> > > level
> > > error casin0 uat <class 'RuntimeError'> Non-thread-safe operation
> invoked
> > > on an event loop other than the current one Traceback (most recent call
> > > last):
> > > File "C:\dev\xxx.py", line 547, in xxx
> > > await asyncio.sleep(algo.param.interval_ms / 1000)
> > > File "C:\ProgramData\Anaconda3\lib\asyncio\tasks.py", line 651, in
> > sleep
> > > h = loop.call_later(delay,
> > > File "C:\ProgramData\Anaconda3\lib\asyncio\base_events.py", line 714,
> > in
> > > call_later
> > > timer = self.call_at(self.time() + delay, callback, *args,
> > > File "C:\ProgramData\Anaconda3\lib\asyncio\base_events.py", line 727,
> > in
> > > call_at
> > > self._check_thread()
> > > File "C:\ProgramData\Anaconda3\lib\asyncio\base_events.py", line 785,
> > in
> > > _check_thread
> > > raise RuntimeError(
> > > RuntimeError: Non-thread-safe operation invoked on an event loop other
> > than
> > > the current one
> > >
> > > On Sat, Feb 26, 2022 at 2:36 PM Andrew Svetlov <report@bugs.python.org
> >
> > > wrote:
> > >
> > > >
> > > > Andrew Svetlov <andrew.svetlov@gmail.com> added the comment:
> > > >
> > > > Also, fix is 1 week old. There are no python releases with the fix
> > > > included yet.
> > > >
> > > > On Sat, Feb 26, 2022, 06:58 Norman Fung <report@bugs.python.org>
> > wrote:
> > > >
> > > > >
> > > > > Norman Fung <norman.lm.fung@gmail.com> added the comment:
> > > > >
> > > > > Thanks Jelle, let me try upgrade first.
> > > > >
> > > > > Norman
> > > > >
> > > > > On Sat, Feb 26, 2022 at 9:27 AM Jelle Zijlstra <
> > report@bugs.python.org
> > > >
> > > > > wrote:
> > > > >
> > > > > >
> > > > > > Jelle Zijlstra <jelle.zijlstra@gmail.com> added the comment:
> > > > > >
> > > > > > 3.8 is only receiving security fixes now. Please upgrade. If you
> > > cannot
> > > > > > upgrade, I suggest manually applying the patch from
> > > > > > https://github.com/python/cpython/pull/31441/files to your
> > > > installation
> > > > > > of Python.
> > > > > >
> > > > > > ----------
> > > > > > nosy: +Jelle Zijlstra
> > > > > > resolution: -> out of date
> > > > > > stage: -> resolved
> > > > > > status: open -> closed
> > > > > >
> > > > > > _______________________________________
> > > > > > Python tracker <report@bugs.python.org>
> > > > > > <https://bugs.python.org/issue46859>
> > > > > > _______________________________________
> > > > > >
> > > > >
> > > > > ----------
> > > > >
> > > > > _______________________________________
> > > > > Python tracker <report@bugs.python.org>
> > > > > <https://bugs.python.org/issue46859>
> > > > > _______________________________________
> > > > >
> > > >
> > > > ----------
> > > >
> > > > _______________________________________
> > > > Python tracker <report@bugs.python.org>
> > > > <https://bugs.python.org/issue46859>
> > > > _______________________________________
> > > >
> > >
> > > ----------
> > >
> > > _______________________________________
> > > Python tracker <report@bugs.python.org>
> > > <https://bugs.python.org/issue46859>
> > > _______________________________________
> > >
> >
> > ----------
> >
> > _______________________________________
> > Python tracker <report@bugs.python.org>
> > <https://bugs.python.org/issue46859>
> > _______________________________________
> >
>
> ----------
>
> _______________________________________
> Python tracker <report@bugs.python.org>
> <https://bugs.python.org/issue46859>
> _______________________________________
> |
➜
