Async exec in python
I would like to call exec in an async function and do something like the following code (which is not valid):
import asyncio
async def f():
await exec('x = 1\n' 'await asyncio.sleep(x)')
More precisely, I would like to be able to wait for the future inside the code that runs in exec.
How can this be achieved?
source to share
Note: F-strings are only supported in Python 3.6+. For older versions, use
%s
,.format()
or classic+
concatenation.
async def aexec(code):
# Make an async function with the code and 'exec' it
exec(
f'async def __ex(): ' +
''.join(f'\n {l}' for l in code.split('\n'))
)
# Get '__ex' from local variables, call it and return the result
return await locals()['__ex']()
Known Issues:
- If you use newlines in the string (triple quotes) it will mess up the formatting.
source to share
The problem is when you are trying to wait for None
object- exec
ignore the return value from your code and always return None
. If you want to execute and wait for the result, you must use eval
- eval
returns the value of the given expression.
Your code should look like this:
import asyncio
async def f():
exec('x = 1')
await eval('asyncio.sleep(x)')
loop = asyncio.get_event_loop()
loop.run_until_complete(f())
loop.close()
source to share
Thanks for all the suggestions. I figured this could be done with green asynchronous lines, as potions allow for a "high wait":
import greenlet
import asyncio
class GreenAwait:
def __init__(self, child):
self.current = greenlet.getcurrent()
self.value = None
self.child = child
def __call__(self, future):
self.value = future
self.current.switch()
def __iter__(self):
while self.value is not None:
yield self.value
self.value = None
self.child.switch()
def gexec(code):
child = greenlet.greenlet(exec)
gawait = GreenAwait(child)
child.switch(code, {'gawait': gawait})
yield from gawait
async def aexec(code):
green = greenlet.greenlet(gexec)
gen = green.switch(code)
for future in gen:
await future
# modified asyncio example from Python docs
CODE = ('import asyncio\n'
'import datetime\n'
'async def display_date():\n'
' for i in range(5):\n'
' print(datetime.datetime.now())\n'
' await asyncio.sleep(1)\n')
def loop():
loop = asyncio.get_event_loop()
loop.run_until_complete(aexec(CODE + 'gawait(display_date())'))
loop.close()
source to share
This is based on @YouTwitFace's answer, but keeps global variables intact, handles local variables better, and passes kwargs. Note that multi-line strings still do not retain their formatting. Perhaps you want this ?
async def aexec(code, **kwargs):
# Don't clutter locals
locs = {}
# Restore globals later
globs = globals().copy()
args = ", ".join(list(kwargs.keys()))
exec(f"async def func({args}):\n " + code.replace("\n", "\n "), {}, locs)
# Don't expect it to return from the coro.
result = await locs["func"](**kwargs)
try:
globals().clear()
# Inconsistent state
finally:
globals().update(**globs)
return result
It starts with rescuing the locals. It declares a function, but with a limited local namespace, so it doesn't affect things declared in the aexec helper. The function is called func
and we get access to the locs
exec locals result. locs["func"]
is what we want to accomplish, so we call it using **kwargs
from the aexec call, which moves those arguments to the local namespace. Then we wait for that and save as result
. Finally, we restore the locals and return the result.
Warning:
Don't use this if there is any multi-threaded code regarding globals. Go to @YouTwitFace's answer which is simpler and thread-safe, or remove the global save / restore code
source to share
Here's a function using AST to do things. This means that multi-line strings will work perfectly and the line numbers will match the original statements. In addition, if the last "product" is an unused expression, it is returned as in Java lambdas.
async def meval(code, **kwargs):
# Note to self: please don't set globals here as they will be lost.
# Don't clutter locals
locs = {}
# Restore globals later
globs = globals().copy()
# This code saves __name__ and __package into a kwarg passed to the function. It is set before the users code runs to make sure $
global_args = "_globs"
while global_args in globs.keys():
# Make sure there no name collision, just keep prepending _s
global_args = "_"+global_args
kwargs[global_args] = {}
for glob in ["__name__", "__package__"]:
# Copy data to args we are sending
kwargs[global_args][glob] = globs[glob]
root = parse(code, 'exec')
code = root.body
if isinstance(code[-1], Expr): # If we can use it as a lambda return (but multiline)
code[-1] = copy_location(code[-1], Return(code[-1].value)) # Change it to a return statement
args = []
for a in list(map(lambda x: arg(x, None), kwargs.keys())):
a.lineno = 0
a.col_offset = 0
args += [a]
fun = AsyncFunctionDef('tmp', arguments(args=[], vararg=None, kwonlyargs=args, kwarg=None, defaults=[], kw_defaults=[None for i $
fun.lineno = 0
fun.col_offset = 0
mod = Module([fun])
comp = compile(mod, '<string>', 'exec')
exec(comp, {}, locs)
r = await locs["tmp"](**kwargs)
try:
globals().clear()
# Inconsistent state
finally:
globals().update(**globs)
return r
Update relative import fixes:
# We dont modify locals VVVV ; this lets us keep the message available to the user-provided function
async def meval(code, **kwargs):
# Note to self: please don't set globals here as they will be lost.
# Don't clutter locals
locs = {}
# Restore globals later
globs = globals().copy()
# This code saves __name__ and __package into a kwarg passed to the function.
# It is set before the users code runs to make sure relative imports work
global_args = "_globs"
while global_args in globs.keys():
# Make sure there no name collision, just keep prepending _s
global_args = "_" + global_args
kwargs[global_args] = {}
for glob in ["__name__", "__package__"]:
# Copy data to args we are sending
kwargs[global_args][glob] = globs[glob]
root = ast.parse(code, 'exec')
code = root.body
if isinstance(code[-1], ast.Expr): # If we can use it as a lambda return (but multiline)
code[-1] = ast.copy_location(ast.Return(code[-1].value), code[-1]) # Change it to a return statement
# globals().update(**<global_args>)
glob_copy = ast.Expr(ast.Call(func=ast.Attribute(value=ast.Call(func=ast.Name(id='globals', ctx=ast.Load()),
args=[], keywords=[]),
attr='update', ctx=ast.Load()),
args=[], keywords=[ast.keyword(arg=None,
value=ast.Name(id=global_args, ctx=ast.Load()))]))
glob_copy.lineno = 0
glob_copy.col_offset = 0
ast.fix_missing_locations(glob_copy)
code.insert(0, glob_copy)
args = []
for a in list(map(lambda x: ast.arg(x, None), kwargs.keys())):
a.lineno = 0
a.col_offset = 0
args += [a]
fun = ast.AsyncFunctionDef('tmp', ast.arguments(args=[], vararg=None, kwonlyargs=args, kwarg=None, defaults=[],
kw_defaults=[None for i in range(len(args))]), code, [], None)
fun.lineno = 0
fun.col_offset = 0
mod = ast.Module([fun])
comp = compile(mod, '<string>', 'exec')
exec(comp, {}, locs)
r = await locs["tmp"](**kwargs)
try:
globals().clear()
# Inconsistent state
finally:
globals().update(**globs)
return r
This is all public domain, but I would appreciate a loan.
source to share