diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -720,9 +720,9 @@ ``'u'`` ``Py_UNICODE`` ``'U'`` ``unicode`` ``'w*'`` ``Py_buffer(types='bytearray rwbuffer')`` -``'y#'`` ``str(type='bytes', length=True)`` +``'y#'`` ``str(types='bytes', length=True)`` ``'Y'`` ``PyByteArrayObject`` -``'y'`` ``str(type='bytes')`` +``'y'`` ``str(types='bytes')`` ``'y*'`` ``Py_buffer`` ``'Z#'`` ``Py_UNICODE(nullable=True, length=True)`` ``'z#'`` ``str(nullable=True, length=True)`` diff --git a/Lib/inspect.py b/Lib/inspect.py --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1969,55 +1969,70 @@ if not isinstance(module, ast.Module): return None - # ast.FunctionDef f = module.body[0] parameters = [] empty = Parameter.empty invalid = object() - def parse_attribute(node): - if not isinstance(node.ctx, ast.Load): - return None + module = None + module_dict = {} + module_name = getattr(func, '__module__', None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + module_dict = module.__dict__ + sys_module_dict = sys.modules - value = node.value - o = parse_node(value) - if o is invalid: - return invalid + def parse_name(node): + assert isinstance(node, ast.arg) + if node.annotation != None: + raise ValueError("Annotations are not currently supported") + return node.arg - if isinstance(value, ast.Name): - name = o - if name not in sys.modules: - return invalid - o = sys.modules[name] + def wrap_value(s): + try: + value = eval(s, module_dict) + except NameError: + try: + value = eval(s, sys_module_dict) + except NameError: + raise RuntimeError() - return getattr(o, node.attr, invalid) + if isinstance(value, str): + return ast.Str(value) + if isinstance(value, (int, float)): + return ast.Num(value) + raise RuntimeError() - def parse_node(node): - if isinstance(node, ast.arg): - if node.annotation != None: - raise ValueError("Annotations are not currently supported") - return node.arg - if isinstance(node, ast.Num): - return node.n - if isinstance(node, ast.Str): - return node.s - if isinstance(node, ast.NameConstant): - return node.value - if isinstance(node, ast.Attribute): - return parse_attribute(node) - if isinstance(node, ast.Name): + class RewriteSymbolics(ast.NodeTransformer): + def visit_Attribute(self, node): + a = [] + n = node + while isinstance(n, ast.Attribute): + a.append(n.attr) + n = n.value + if not isinstance(n, ast.Name): + raise RuntimeError() + a.append(n.id) + value = ".".join(reversed(a)) + return wrap_value(value) + + def visit_Name(self, node): if not isinstance(node.ctx, ast.Load): - return invalid - return node.id - return invalid + raise ValueError() + return wrap_value(node.id) - def p(name_node, default_node, default=empty): - name = parse_node(name_node) + def p(name_node, default_node, default=object()): + name = parse_name(name_node) if name is invalid: return None if default_node: - o = parse_node(default_node) + try: + default_node = RewriteSymbolics().visit(default_node) + o = ast.literal_eval(default_node) + except ValueError: + o = invalid if o is invalid: return None default = o if o is not invalid else default diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -1603,10 +1603,12 @@ self.assertEqual(p('s'), 'avocado') self.assertEqual(p('d'), 3.14) self.assertEqual(p('i'), 35) - self.assertEqual(p('c'), sys.maxsize) self.assertEqual(p('n'), None) self.assertEqual(p('t'), True) self.assertEqual(p('f'), False) + self.assertEqual(p('local'), 3) + self.assertEqual(p('sys'), sys.maxsize) + self.assertEqual(p('exp'), sys.maxsize - 1) def test_signature_on_non_function(self): with self.assertRaisesRegex(TypeError, 'is not a callable object'): diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -450,7 +450,7 @@ flags: str="r" How to open the file. "r" for reading, "w" for writing, etc. - mode: int(doc_default="0o666") = 0o666 + mode: int(py_default="0o666") = 0o666 If creating a new file, the mode bits for the new file (e.g. os.O_RDWR). diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2869,8 +2869,8 @@ "This docstring has a valid signature and some extra newlines." ); -PyDoc_STRVAR(docstring_with_signature_with_defaults, -"docstring_with_signature_with_defaults(s='avocado', d=3.14, i=35, c=sys.maxsize, n=None, t=True, f=False)\n" +PyDoc_STRVAR(docstring_with_signature_with_defaults, /* c=sys.maxsize, exp=sys.maxsize - 1 */ +"docstring_with_signature_with_defaults(s='avocado', d=3.14, i=35, n=None, t=True, f=False, local=the_number_three, sys=sys.maxsize, exp=sys.maxsize - 1)\n" "\n" "\n" "\n" @@ -3317,6 +3317,8 @@ Py_INCREF(&PyInstanceMethod_Type); PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); + PyModule_AddIntConstant(m, "the_number_three", 3); + TestError = PyErr_NewException("_testcapi.error", NULL, NULL); Py_INCREF(TestError); PyModule_AddObject(m, "error", TestError); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2401,7 +2401,7 @@ /*[clinic input] -os.stat -> object(doc_default='stat_result') +os.stat path : path_t(allow_fd=True) Path to be examined; can be string, bytes, or open-file-descriptor int. @@ -2523,7 +2523,7 @@ #define OS_ACCESS_DIR_FD_CONVERTER dir_fd_unavailable #endif /*[clinic input] -os.access -> object(doc_default='True if granted, False otherwise') +os.access path: path_t(allow_fd=True) Path to be tested; can be string, bytes, or open-file-descriptor int. diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -54,6 +54,13 @@ NULL = Null() +class Unknown: + def __repr__(self): + return '' + +unknown = Unknown() + + def _text_accumulator(): text = [] def output(): @@ -1338,29 +1345,7 @@ def is_keyword_only(self): return self.kind == inspect.Parameter.KEYWORD_ONLY -py_special_values = { - NULL: "None", -} - -def py_repr(o): - special = py_special_values.get(o) - if special: - return special - return repr(o) - - -c_special_values = { - NULL: "NULL", - None: "Py_None", -} - -def c_repr(o): - special = c_special_values.get(o) - if special: - return special - if isinstance(o, str): - return '"' + quoted_for_c_string(o) + '"' - return repr(o) + def add_c_converter(f, name=None): if not name: @@ -1402,8 +1387,7 @@ """ For the init function, self, name, function, and default must be keyword-or-positional parameters. All other - parameters (including "required" and "doc_default") - must be keyword-only. + parameters must be keyword-only. """ # The C type to use for this variable. @@ -1413,23 +1397,23 @@ # The Python default value for this parameter, as a Python value. # Or the magic value "unspecified" if there is no default. + # Or the magic value "unknown" if this value is a cannot be evaluated + # at Argument-Clinic-preprocessing time (but is presumed to be valid + # at runtime). default = unspecified # If not None, default must be isinstance() of this type. # (You can also specify a tuple of types.) default_type = None - # "default" as it should appear in the documentation, as a string. - # Or None if there is no default. - doc_default = None - - # "default" converted into a str for rendering into Python code. - py_default = None - # "default" converted into a C value, as a string. # Or None if there is no default. c_default = None + # "default" converted into a Python value, as a string. + # Or None if there is no default. + py_default = None + # The default value used to initialize the C variable when # there is no default, but not specifying a default may # result in an "uninitialized variable" warning. This can @@ -1480,12 +1464,12 @@ # Only used by format units ending with '#'. length = False - def __init__(self, name, function, default=unspecified, *, doc_default=None, c_default=None, py_default=None, required=False, annotation=unspecified, **kwargs): + def __init__(self, name, function, default=unspecified, *, c_default=None, py_default=None, annotation=unspecified, **kwargs): self.function = function self.name = name if default is not unspecified: - if self.default_type and not isinstance(default, self.default_type): + if self.default_type and not isinstance(default, (self.default_type, Unknown)): if isinstance(self.default_type, type): types_str = self.default_type.__name__ else: @@ -1493,23 +1477,19 @@ fail("{}: default value {!r} for field {} is not of type {}".format( self.__class__.__name__, default, name, types_str)) self.default = default - self.py_default = py_default if py_default is not None else py_repr(default) - self.doc_default = doc_default if doc_default is not None else self.py_default - self.c_default = c_default if c_default is not None else c_repr(default) - else: - self.py_default = py_default - self.doc_default = doc_default - self.c_default = c_default + + self.c_default = c_default + self.py_default = py_default + if annotation != unspecified: fail("The 'annotation' parameter is not currently permitted.") - self.required = required self.converter_init(**kwargs) def converter_init(self): pass def is_optional(self): - return (self.default is not unspecified) and (not self.required) + return (self.default is not unspecified) def render(self, parameter, data): """ @@ -1660,7 +1640,7 @@ c_ignored_default = "'\0'" def converter_init(self): - if len(self.default) != 1: + if isinstance(self.default, str) and (len(self.default) != 1): fail("char_converter: illegal default value " + repr(self.default)) @@ -1988,8 +1968,8 @@ # Or the magic value "unspecified" if there is no default. default = None - def __init__(self, *, doc_default=None, **kwargs): - self.doc_default = doc_default + def __init__(self, *, py_default=None, **kwargs): + self.py_default = py_default try: self.return_converter_init(**kwargs) except TypeError as e: @@ -2239,15 +2219,18 @@ self.block.signatures.append(c) def at_classmethod(self): - assert self.kind is CALLABLE + if self.kind is not CALLABLE: + fail("Can't set @classmethod, function is not a normal callable") self.kind = CLASS_METHOD def at_staticmethod(self): - assert self.kind is CALLABLE + if self.kind is not CALLABLE: + fail("Can't set @staticmethod, function is not a normal callable") self.kind = STATIC_METHOD def at_coexist(self): - assert self.coexist == False + if self.coexist: + fail("Called @coexist twice!") self.coexist = True @@ -2496,48 +2479,120 @@ else: fail("Function " + self.function.name + " has an unsupported group configuration. (Unexpected state " + str(self.parameter_state) + ")") - ast_input = "def x({}): pass".format(line) + base, equals, default = line.rpartition('=') + if not equals: + base = default + default = None module = None try: + ast_input = "def x({}): pass".format(base) module = ast.parse(ast_input) except SyntaxError: - pass + try: + default = None + ast_input = "def x({}): pass".format(line) + module = ast.parse(ast_input) + except SyntaxError: + pass if not module: fail("Function " + self.function.name + " has an invalid parameter declaration:\n\t" + line) function_args = module.body[0].args parameter = function_args.args[0] - py_default = None - parameter_name = parameter.arg name, legacy, kwargs = self.parse_converter(parameter.annotation) - if function_args.defaults: - expr = function_args.defaults[0] - # mild hack: explicitly support NULL as a default value - if isinstance(expr, ast.Name) and expr.id == 'NULL': - value = NULL - elif isinstance(expr, ast.Attribute): + if not default: + value = unspecified + if 'py_default' in kwargs: + fail("You can't specify py_default without specifying a default value!") + else: + default = default.strip() + ast_input = "x = {}".format(default) + try: + module = ast.parse(ast_input) + + # blacklist of disallowed ast nodes + class DetectBadNodes(ast.NodeVisitor): + bad = False + def bad_node(self, node): + self.bad = True + + # inline function call + visit_Call = bad_node + # inline if statement ("x = 3 if y else z") + visit_IfExp = bad_node + + # comprehensions and generator expressions + visit_ListComp = visit_SetComp = bad_node + visit_DictComp = visit_GeneratorExp = bad_node + + # literals for advanced types + visit_Dict = visit_Set = bad_node + visit_List = visit_Tuple = bad_node + visit_Ellipsis = bad_node + + # "starred": "a = [1, 2, 3]; *a" + visit_Starred = bad_node + + blacklist = DetectBadNodes() + blacklist.visit(module) + if blacklist.bad: + fail("Disallowed expression as default value: " + repr(default)) + + expr = module.body[0].value + # mild hack: explicitly support NULL as a default value + if isinstance(expr, ast.Name) and expr.id == 'NULL': + value = NULL + py_default = 'None' + c_default = "NULL" + elif isinstance(expr, (ast.BinOp, ast.UnaryOp)): + c_default = kwargs.get("c_default") + if not (isinstance(c_default, str) and c_default): + fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") + py_default = default + value = unknown + elif isinstance(expr, ast.Attribute): + a = [] + n = expr + while isinstance(n, ast.Attribute): + a.append(n.attr) + n = n.value + if not isinstance(n, ast.Name): + fail("Malformed default value (looked like a Python constant)") + a.append(n.id) + py_default = ".".join(reversed(a)) + + c_default = kwargs.get("c_default") + if not (isinstance(c_default, str) and c_default): + fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") + + try: + value = eval(py_default) + except NameError: + value = unknown + else: + value = ast.literal_eval(expr) + py_default = repr(value) + if isinstance(value, (bool, None.__class__)): + c_default = "Py_" + py_default + elif isinstance(value, str): + c_default = '"' + quoted_for_c_string(value) + '"' + else: + c_default = py_default + + except SyntaxError as e: + fail("Syntax error: " + repr(e.text)) + except (ValueError, AttributeError): + value = unknown c_default = kwargs.get("c_default") + py_default = default if not (isinstance(c_default, str) and c_default): fail("When you specify a named constant (" + repr(py_default) + ") as your default value,\nyou MUST specify a valid c_default.") - a = [] - n = expr - while isinstance(n, ast.Attribute): - a.append(n.attr) - n = n.value - if not isinstance(n, ast.Name): - fail("Malformed default value (looked like a Python constant)") - a.append(n.id) - py_default = ".".join(reversed(a)) - kwargs["py_default"] = py_default - value = eval(py_default) - else: - value = ast.literal_eval(expr) - else: - value = unspecified + kwargs.setdefault('c_default', c_default) + kwargs.setdefault('py_default', py_default) dict = legacy_converters if legacy else converters legacy_str = "legacy " if legacy else "" @@ -2726,7 +2781,7 @@ if p.converter.is_optional(): a.append('=') value = p.converter.default - a.append(p.converter.doc_default) + a.append(p.converter.py_default) s = fix_right_bracket_count(p.right_bracket_count) s += "".join(a) if add_comma: @@ -2737,9 +2792,18 @@ add(fix_right_bracket_count(0)) add(')') - # if f.return_converter.doc_default: + # PEP 8 says: + # + # The Python standard library will not use function annotations + # as that would result in a premature commitment to a particular + # annotation style. Instead, the annotations are left for users + # to discover and experiment with useful annotation styles. + # + # therefore this is commented out: + # + # if f.return_converter.py_default: # add(' -> ') - # add(f.return_converter.doc_default) + # add(f.return_converter.py_default) docstring_first_line = output() @@ -2947,8 +3011,8 @@ # print(" ", short_name + "".join(parameters)) print() - print("All converters also accept (doc_default=None, required=False, annotation=None).") - print("All return converters also accept (doc_default=None).") + print("All converters also accept (c_default=None, py_default=None, annotation=None).") + print("All return converters also accept (py_default=None).") sys.exit(0) if ns.make: diff --git a/Tools/clinic/clinic_test.py b/Tools/clinic/clinic_test.py --- a/Tools/clinic/clinic_test.py +++ b/Tools/clinic/clinic_test.py @@ -233,20 +233,20 @@ self._test_clinic(""" verbatim text here lah dee dah -/*[copy] +/*[copy input] def -[copy]*/ +[copy start generated code]*/ abc -/*[copy checksum: 03cfd743661f07975fa2f1220c5194cbaff48451]*/ +/*[copy end generated code: checksum=03cfd743661f07975fa2f1220c5194cbaff48451]*/ xyz """, """ verbatim text here lah dee dah -/*[copy] +/*[copy input] def -[copy]*/ +[copy start generated code]*/ def -/*[copy checksum: 7b18d017f89f61cf17d47f92749ea6930a3f1deb]*/ +/*[copy end generated code: checksum=7b18d017f89f61cf17d47f92749ea6930a3f1deb]*/ xyz """) @@ -294,17 +294,6 @@ p = function.parameters['path'] self.assertEqual(1, p.converter.args['allow_fd']) - def test_param_docstring(self): - function = self.parse_function(""" -module os -os.stat as os_stat_fn -> object(doc_default='stat_result') - - path: str - Path to be examined""") - p = function.parameters['path'] - self.assertEqual("Path to be examined", p.docstring) - self.assertEqual(function.return_converter.doc_default, 'stat_result') - def test_function_docstring(self): function = self.parse_function(""" module os