diff -r ad51ed93377c -r 3d0ca1e82c46 Doc/library/ast.rst --- a/Doc/library/ast.rst Thu Oct 11 00:11:26 2012 -0700 +++ b/Doc/library/ast.rst Thu Oct 11 16:47:49 2012 +0100 @@ -126,6 +126,32 @@ .. versionchanged:: 3.2 Now allows bytes and set literals. +.. function:: lookup_eval(node_or_string, context, allow_imports=False) + + Safely evaluate an expression node or a string containing a Python expression. + In addition to the literals supported by :func:`literal_eval`, ``lookup_eval`` + supports the following features: + + * The use of names, which are looked up in a mapping, ``context``. In + addition, the names ``True``, ``False`` and ``None`` are supported. If + ``allow_imports`` is ``True`` and a name is not found in ``context``, an + attempt will be made to import it and use the imported module as the + result. + + * The use of the following sub-expression types: + + * Array indexing and slicing + * Attribute access + * Arithmetic operators: ``+``, ``-``, ``*``, ``/``, ``//``, ``%``, ``**`` + * Bitwise operators: ``&``, ````|, ``^`` ``<<``, ``>>`` + * Relational operators: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=`` + * Membership: ``in``, ``not in`` + * Logic: ``and``, ``or`` + * Unary operators: ``+``, ``-``, ``~``, ``not`` + + On any error (e.g. a syntax error), a :class:`ValueError` is raised. + + .. versionadded:: 3.4 .. function:: get_docstring(node, clean=True) diff -r ad51ed93377c -r 3d0ca1e82c46 Lib/ast.py --- a/Lib/ast.py Thu Oct 11 00:11:26 2012 -0700 +++ b/Lib/ast.py Thu Oct 11 16:47:49 2012 +0100 @@ -315,3 +315,205 @@ else: setattr(node, field, new_node) return node + + +class Evaluator(object): + builtin_names = { + 'None': None, + 'False': False, + 'True': True, + } + + operators = { + 'add': lambda x, y: x + y, + 'bitand': lambda x, y: x & y, + 'bitor': lambda x, y: x | y, + 'bitxor': lambda x, y: x ^ y, + 'div': lambda x, y: x / y, + 'eq': lambda x, y: x == y, + 'floordiv': lambda x, y: x // y, + 'gt': lambda x, y: x > y, + 'gte': lambda x, y: x >= y, + 'in': lambda x, y: x in y, + 'invert': lambda x: ~x, + 'lshift': lambda x, y: x << y, + 'lt': lambda x, y: x < y, + 'lte': lambda x, y: x <= y, + 'mod': lambda x, y: x % y, + 'mult': lambda x, y: x * y, + 'not': lambda x: not x, + 'noteq': lambda x, y: x != y, + 'notin': lambda x, y: x not in y, + 'pow': lambda x, y: x ** y, + 'rshift': lambda x, y: x >> y, + 'sub': lambda x, y: x - y, + 'uadd': lambda x: +x, + 'usub': lambda x: -x, + } + + handlers = {} + + def __init__(self, context=None, allow_imports=False): + self.context = context or {} + self.source = None + self.allow_imports = allow_imports + + def get_fragment(self, offset): + fragment_len = 10 + s = 'at position %d: %r' % (offset, self.source[offset:offset + fragment_len]) + if offset + fragment_len < len(self.source): + s += '...' + return s + + def evaluate(self, node, filename=None): + if isinstance(node, str): + self.source = node + kwargs = dict(mode='eval') + if filename: + kwargs['filename'] = filename + try: + node = parse(node, **kwargs) + except SyntaxError as e: + s = self.get_fragment(e.offset) + raise ValueError('syntax error %s' % s) + node_type = node.__class__.__name__.lower() + if node_type in self.handlers: + handler = self.handlers[node_type] + else: + handler = getattr(self, 'do_%s' % node_type, None) + if handler is None: + if self.source is None: + s = '(source not available)' + else: + s = self.get_fragment(node.col_offset) + raise ValueError("don't know how to evaluate %r %s" % ( + node_type, s)) + return handler(node) + + def do_attribute(self, node): + container = self.evaluate(node.value) + return getattr(container, node.attr) + + def do_binop(self, node): + op = node.op.__class__.__name__.lower() + if op not in self.operators: + raise ValueError('unsupported operation: %r' % op) + lhs = self.evaluate(node.left) + rhs = self.evaluate(node.right) + return self.operators[op](lhs, rhs) + + def do_boolop(self, node): + result = self.evaluate(node.values[0]) + is_or = node.op.__class__ is Or + is_and = node.op.__class__ is And + assert is_or or is_and + if (is_and and result) or (is_or and not result): + for n in node.values[1:]: + result = self.evaluate(n) + if (is_or and result) or (is_and and not result): + break + return result + + def do_compare(self, node): + lhs = self.evaluate(node.left) + result = True + for op, right in zip(node.ops, node.comparators): + op = op.__class__.__name__.lower() + if op not in self.operators: + raise ValueError('unsupported operation: %r' % op) + rhs = self.evaluate(right) + result = self.operators[op](lhs, rhs) + if not result: + break + lhs = rhs + return result + + def do_dict(self, node): + e = self.evaluate + return dict((e(k), e(v)) for k, v in zip(node.keys, node.values)) + + def do_ellipsis(self, node): + return __builtins__['Ellipsis'] + + def do_expr(self, node): + return self.evaluate(node.value) + + do_index = do_expr + + def do_expression(self, node): + return self.evaluate(node.body) + + def do_extslice(self, node): + e = self.evaluate + return tuple((e(n) for n in node.dims)) + + def do_list(self, node): + return list([self.evaluate(n) for n in node.elts]) + + def do_set(self, node): + return set([self.evaluate(n) for n in node.elts]) + + def do_name(self, node): + if node.id in self.builtin_names: + result = self.builtin_names[node.id] + elif node.id in self.context: + result = self.context[node.id] + else: + if not self.allow_imports: + raise ValueError('unknown name: %r' % node.id) + try: + result = __import__(node.id) + except ImportError: + raise ValueError('unknown name: %r' % node.id) + return result + + def do_num(self, node): + return node.n + + def do_slice(self, node): + if node.lower is None: + lower = None + else: + lower = self.evaluate(node.lower) + if node.upper is None: + upper = None + else: + upper = self.evaluate(node.upper) + if node.step is None: + step = None + else: + step = self.evaluate(node.step) + return __builtins__['slice'](lower, upper, step) + + def do_str(self, node): + return node.s + + do_bytes = do_str + + def do_subscript(self, node): + assert node.ctx.__class__ is Load + val = self.evaluate(node.value) + if not isinstance(node.slice, (Index, Slice, Ellipsis, ExtSlice)): + raise ValueError('Unable to get subscript: %r', + node.slice.__class__.__name__) + indices = self.evaluate(node.slice) + if isinstance(node.slice, ExtSlice): + result = val[(indices)] + else: + result = val.__getitem__(indices) + return result + + def do_tuple(self, node): + return tuple([self.evaluate(n) for n in node.elts]) + + def do_unaryop(self, node): + op = node.op.__class__.__name__.lower() + operand = self.evaluate(node.operand) + if op not in self.operators: + raise ValueError('unsupported operation: %r' % op) + return self.operators[op](operand) + +evaluator = Evaluator + +def lookup_eval(source, context, allow_imports=False): + return evaluator(context, allow_imports=allow_imports).evaluate(source, '') diff -r ad51ed93377c -r 3d0ca1e82c46 Lib/logging/config.py --- a/Lib/logging/config.py Thu Oct 11 00:11:26 2012 -0700 +++ b/Lib/logging/config.py Thu Oct 11 16:47:49 2012 +0100 @@ -24,6 +24,7 @@ To use, simply 'import logging' and log away! """ +from ast import lookup_eval import sys, logging, logging.handlers, socket, struct, traceback, re import io @@ -135,12 +136,14 @@ section = cp["handler_%s" % hand] klass = section["class"] fmt = section.get("formatter", "") + context = dict(vars(logging)) + context['handlers'] = logging.handlers try: - klass = eval(klass, vars(logging)) - except (AttributeError, NameError): + klass = lookup_eval(klass, context, True) + except ValueError: klass = _resolve(klass) args = section["args"] - args = eval(args, vars(logging)) + args = lookup_eval(args, context, True) h = klass(*args) if "level" in section: level = section["level"] diff -r ad51ed93377c -r 3d0ca1e82c46 Lib/test/test_ast.py --- a/Lib/test/test_ast.py Thu Oct 11 00:11:26 2012 -0700 +++ b/Lib/test/test_ast.py Thu Oct 11 16:47:49 2012 +0100 @@ -526,6 +526,173 @@ compile(mod, 'test', 'exec') self.assertIn("invalid integer value: None", str(cm.exception)) + def test_literal_eval_issue4907(self): + self.assertEqual(ast.literal_eval('2j'), 2j) + self.assertEqual(ast.literal_eval('10 + 2j'), 10 + 2j) + self.assertEqual(ast.literal_eval('1.5 - 2j'), 1.5 - 2j) + + def test_bad_integer(self): + # issue13436: Bad error message with invalid numeric values + body = [ast.ImportFrom(module='time', + names=[ast.alias(name='sleep')], + level=None, + lineno=None, col_offset=None)] + mod = ast.Module(body) + with self.assertRaises(ValueError) as cm: + compile(mod, 'test', 'exec') + self.assertIn("invalid integer value: None", str(cm.exception)) + + def test_lookup_eval_no_context(self): + compute = lambda s: ast.lookup_eval(s, {}, False) + compute_import = lambda s: ast.lookup_eval(s, {}, True) + + # Do the same tests as for literal_eval + + self.assertEqual(compute('[1, 2, 3]'), [1, 2, 3]) + self.assertEqual(compute('{"foo": 42}'), {"foo": 42}) + self.assertEqual(compute('(True, False, None)'), (True, False, None)) + self.assertEqual(compute('{1, 2, 3}'), {1, 2, 3}) + self.assertEqual(compute('b"hi"'), b"hi") + self.assertRaises(ValueError, compute, 'foo()') + self.assertEqual(compute('-6'), -6) + self.assertEqual(compute('-6j+3'), 3-6j) + self.assertEqual(compute('3.25'), 3.25) + self.assertEqual(compute('2j'), 2j) + self.assertEqual(compute('10 + 2j'), 10 + 2j) + self.assertEqual(compute('1.5 - 2j'), 1.5 - 2j) + + # Add some for builtin and imported names + self.assertEqual(compute_import('ast'), ast) + self.assertEqual(compute_import('None'), None) + self.assertEqual(compute_import('True'), True) + self.assertEqual(compute_import('False'), False) + self.assertIs(compute_import('...'), Ellipsis) + + self.assertRaises(ValueError, compute, 'ast') + + def test_lookup_eval_with_context(self): + context = { + 'v1': [1, 2, 3], + 'v2': {"foo": 42}, + 'v3': (True, False, None), + 'v4': {1, 2, 3}, + 'v5': b"hi", + 'v6': 3 - 6j, + 'v7': 3.25, + 'v8': {1, 3, 5}, + 'v9': 1.0, + 'v10': 1 + 2j, + 'v11': (None, True, False), + 'v12': b' there', + 'v13': 'ahoy ', + 'v14': 'matey', + 'v15': [4, 5, 6], + 'v16': 0x2222, + 'v17': 0x1111, + 'v18': 2, + 'v19': 4, + 'v20': False, + 'v21': True, + 'None': 'not really None', + 'Ellipsis': 'not really Ellipsis', + } + compute = lambda s: ast.lookup_eval(s, context, False) + compute_import = lambda s: ast.lookup_eval(s, context, True) + + # Test name lookup + self.assertEqual(compute('v1'), [1, 2, 3]) + self.assertEqual(compute('None'), None) + self.assertEqual(compute('False'), False) + self.assertEqual(compute('True'), True) + self.assertIs(compute('...'), Ellipsis) + + # Test addition operator + self.assertEqual(compute('v1 + v15'), [1, 2, 3, 4, 5, 6]) + self.assertEqual(compute('v3 + v11'), (True, False, None, + None, True, False)) + self.assertEqual(compute('v7 + v9'), 4.25) + self.assertEqual(compute('v6 + v10'), 4 - 4j) + self.assertEqual(compute('v5 + v12'), b'hi there') + self.assertEqual(compute('v13 + v14'), 'ahoy matey') + + # Test subtraction operator + self.assertEqual(compute('v7 - v9'), 2.25) + self.assertEqual(compute('v6 - v10'), 2 - 8j) + self.assertEqual(compute('v4 - v8'), {2}) + + # Test multiplication operator + self.assertEqual(compute('v18 * v19'), 8) + self.assertEqual(compute('v5 * v18'), b'hihi') + self.assertEqual(compute('v1 * v18'), [1, 2, 3, 1, 2, 3]) + + # Test division operator + self.assertEqual(compute('v19 / v18'), 2) + self.assertEqual(compute('v17 / v18'), 2184.5) + self.assertEqual(compute('v17 // v18'), 2184) + + # Test modulo operator + self.assertEqual(compute('v17 % v19'), 1) + self.assertEqual(compute('"%d" % v19'), '4') + + # Test power operator + self.assertEqual(compute('v18 ** v19'), 16) + self.assertEqual(compute('v7 ** v18'), 10.5625) + + # Test bitwise operators + self.assertEqual(compute('v16 | v17'), 0x3333) + self.assertEqual(compute('v4 | v8'), {1, 2, 3, 5}) + self.assertEqual(compute('v16 & v17'), 0) + self.assertEqual(compute('v4 & v8'), {1, 3}) + self.assertEqual(compute('v16 ^ v17'), 0x3333) + self.assertEqual(compute('v4 ^ v8'), {2, 5}) + self.assertEqual(compute('v16 << v18'), 0x2222 << 2) + self.assertEqual(compute('v16 >> v18'), 0x2222 >> 2) + + # Test relational operators + self.assertEqual(compute('v18 == v18'), True) + self.assertEqual(compute('v18 == v19'), False) + self.assertEqual(compute('v18 != v18'), False) + self.assertEqual(compute('v18 != v19'), True) + self.assertEqual(compute('v18 < v19'), True) + self.assertEqual(compute('v18 <= v19'), True) + self.assertEqual(compute('v19 <= v19'), True) + self.assertEqual(compute('v19 > v18'), True) + self.assertEqual(compute('v19 >= v18'), True) + self.assertEqual(compute('v19 >= v19'), True) + + # Test membership operators + self.assertEqual(compute('v18 in v1'), True) + self.assertEqual(compute('v19 in v1'), False) + self.assertEqual(compute('v18 not in v1'), False) + self.assertEqual(compute('v19 not in v1'), True) + self.assertEqual(compute('v18 in v4'), True) + + # Test unary operators + self.assertEqual(compute('-v18'), -2) + self.assertEqual(compute('+v19'), 4) + self.assertEqual(compute('not v20'), True) + self.assertEqual(compute('~v18'), -3) + + # Test attribute access + self.assertIs(compute_import('ast.parse'), ast.parse) + + # Test array indexing and slicing + self.assertEqual(compute('v1[v18]'), 3) + self.assertEqual(compute('v1[::-1]'), [3, 2, 1]) + self.assertEqual(compute('"abc"[::-1]'), "cba") + + # Test logical operators + self.assertEqual(compute('v20 or v21'), True) + self.assertEqual(compute('v20 and v21'), False) + self.assertEqual(compute('v21 or v20'), True) + self.assertEqual(compute('v21 and v20'), False) + + # Error cases + self.assertRaises(TypeError, compute, 'v4 + v8') # can't add sets + self.assertRaises(ValueError, compute, 'v4 + ') # bad syntax + # function calls are not supported ... + self.assertRaises(ValueError, compute_import, 'cgi.print_directory()') + self.assertRaises(ValueError, compute, 'v99') # unknown name class ASTValidatorTests(unittest.TestCase):