diff -r 8a364deb0225 Lib/argparse.py --- a/Lib/argparse.py Thu May 02 10:44:04 2013 -0700 +++ b/Lib/argparse.py Wed May 22 20:02:11 2013 -0700 @@ -92,6 +92,20 @@ from gettext import gettext as _, ngettext +import logging as _logging +_logging.basicConfig(filename='issue9338.log',level=_logging.DEBUG) + +def _log(*args): + pass +_log = print +def _log(*args): + #_logging.info(args) + try: + args = ' '+' '.join(['%s'%(x,) for x in args]) + except AttributeError: + pass + _logging.info(args) + SUPPRESS = '==SUPPRESS==' @@ -1820,7 +1834,7 @@ action(self, namespace, argument_values, option_string) # function to convert arg_strings into an optional action - def consume_optional(start_index): + def consume_optional(start_index, no_action=False, penult=-1): # get the optional identified at this index option_tuple = option_string_indices[start_index] @@ -1880,6 +1894,28 @@ start = start_index + 1 selected_patterns = arg_strings_pattern[start:] arg_count = match_argument(action, selected_patterns) + + # if action takes a variable number of arguments, see + # if it needs to share any with remaining positionals + _log(action.dest, arg_count, selected_patterns, selected_patterns.count('O')) + if self._is_nargs_variable(action): + # variable range of args for this action + slots = self._match_arguments_partial([action]+positionals, selected_patterns) + _log(' opt+pos slots',slots) + shared_count = slots[0] + else: + shared_count = None + + # penult controls whether this uses this shared_count + # the last optional (ultimate) usually can share + # but earlier ones (penult) might also + + if shared_count is not None and selected_patterns.count('O')<=penult: + # _log(' COUNTS:',arg_count, shared_count) + if arg_count>shared_count: + _log(' changing arg_count %s to shared_count %s'%(arg_count,shared_count)) + arg_count = shared_count + stop = start + arg_count args = arg_strings[start:stop] action_tuples.append((action, args, option_string)) @@ -1887,6 +1923,8 @@ # add the Optional to the list and return the index at which # the Optional's string args stopped + if no_action: + return stop assert action_tuples for action, args, option_string in action_tuples: take_action(action, args, option_string) @@ -1897,7 +1935,7 @@ positionals = self._get_positional_actions() # function to convert arg_strings into positional actions - def consume_positionals(start_index): + def consume_positionals(start_index, no_action=False): # match as many Positionals as possible match_partial = self._match_arguments_partial selected_pattern = arg_strings_pattern[start_index:] @@ -1908,54 +1946,95 @@ for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] start_index += arg_count - take_action(action, args) + if not no_action: + take_action(action, args) # slice off the Positionals that we just parsed and return the # index at which the Positionals' string args stopped positionals[:] = positionals[len(arg_counts):] return start_index - # consume Positionals and Optionals alternately, until we have - # passed the last option string + def consume_loop(no_action=False, penult=-1): + + # consume Positionals and Optionals alternately, until we have + # passed the last option string + + start_index = 0 + if option_string_indices: + max_option_string_index = max(option_string_indices) + else: + max_option_string_index = -1 + + while start_index <= max_option_string_index: + # consume any Positionals preceding the next option + next_option_string_index = min([ + index + for index in option_string_indices + if index >= start_index]) + if start_index != next_option_string_index: + positionals_end_index = consume_positionals(start_index,no_action) + + # only try to parse the next optional if we didn't consume + # the option string during the positionals parsing + if positionals_end_index > start_index: + start_index = positionals_end_index + continue + else: + start_index = positionals_end_index + + # if we consumed all the positionals we could and we're not + # at the index of an option string, there were extra arguments + if start_index not in option_string_indices: + strings = arg_strings[start_index:next_option_string_index] + extras.extend(strings) + start_index = next_option_string_index + + # consume the next optional and any arguments for it + start_index = consume_optional(start_index, no_action, penult) + # _log(start_index,len(positionals),extras) + # consume any positionals following the last Optional + stop_index = consume_positionals(start_index,no_action) + # _log(start_index,len(positionals),extras) + + # if we didn't consume all the argument strings, there were extras + extras.extend(arg_strings[stop_index:]) + return extras + + + penult = arg_strings_pattern.count('O') # # of 'O' in 'AOAA' patttern + opt_actions = [v[0] for v in option_string_indices.values() if v[0]] + _log(arg_strings_pattern, penult, + {'%s%s'%(v.dest,(v.nargs if v.nargs else '')) for v in opt_actions}, + ['%s%s'%(k.dest,(k.nargs if k.nargs else '')) for k in positionals]) + + _cnt = 0 + if self._is_nargs_variable(opt_actions) and positionals and penult>1: + # if there are positionals and one or more 'variable' optionals + # do test loops to see when to start sharing + # test loops + for ii in range(0, penult): + extras = [] + positionals = self._get_positional_actions() + extras = consume_loop(True, ii) + _cnt += 1 + if len(positionals)==0: + _log(' PENULT',ii, (extras if extras else ''), 'all pos matched') + break + else: + _log(' PENULT',ii, + (extras if extras else ''), '%s pos left'%len(positionals)) + if positionals: + _log(' Positionals after penult') + else: + # don't need a test run; but do use action+positionals parsing + ii = 0 + # now the real parsing loop, that takes action extras = [] - start_index = 0 - if option_string_indices: - max_option_string_index = max(option_string_indices) - else: - max_option_string_index = -1 - while start_index <= max_option_string_index: - - # consume any Positionals preceding the next option - next_option_string_index = min([ - index - for index in option_string_indices - if index >= start_index]) - if start_index != next_option_string_index: - positionals_end_index = consume_positionals(start_index) - - # only try to parse the next optional if we didn't consume - # the option string during the positionals parsing - if positionals_end_index > start_index: - start_index = positionals_end_index - continue - else: - start_index = positionals_end_index - - # if we consumed all the positionals we could and we're not - # at the index of an option string, there were extra arguments - if start_index not in option_string_indices: - strings = arg_strings[start_index:next_option_string_index] - extras.extend(strings) - start_index = next_option_string_index - - # consume the next optional and any arguments for it - start_index = consume_optional(start_index) - - # consume any positionals following the last Optional - stop_index = consume_positionals(start_index) - - # if we didn't consume all the argument strings, there were extras - extras.extend(arg_strings[stop_index:]) + positionals = self._get_positional_actions() + extras = consume_loop(False, ii) + _cnt += 1 + _log(' II', _cnt, penult, arg_strings_pattern, extras, len(positionals)) + # make sure all required actions were present and also convert # action defaults which were not given as arguments @@ -1996,6 +2075,7 @@ self.error(msg % ' '.join(names)) # return the updated namespace and the extra arguments + _log(namespace,extras);_log('') return namespace, extras def _read_args_from_files(self, arg_strings): @@ -2207,6 +2287,14 @@ # return the pattern return nargs_pattern + def _is_nargs_variable(self, action): + # return true if action, or any action in a list, takes variable number of args + if isinstance(action,list): + return any(self._is_nargs_variable(a) for a in action) + else: + return action.nargs in [OPTIONAL, ZERO_OR_MORE, ONE_OR_MORE, REMAINDER, PARSER] + # possible change if '{m,n}' is added + # ======================== # Value conversion methods # ======================== @@ -2356,6 +2444,7 @@ def exit(self, status=0, message=None): if message: self._print_message(message, _sys.stderr) + _log(message) _sys.exit(status) def error(self, message): diff -r 8a364deb0225 Lib/test/test_argparse.py --- a/Lib/test/test_argparse.py Thu May 02 10:44:04 2013 -0700 +++ b/Lib/test/test_argparse.py Wed May 22 20:02:11 2013 -0700 @@ -1228,6 +1228,7 @@ class TestNargsZeroOrMore(ParserTestCase): """Tests specifying an args for an Optional that accepts zero or more""" + """Strings that work without the issue9338 patch """ argument_signatures = [Sig('-x', nargs='*'), Sig('y', nargs='*')] failures = [] @@ -1242,6 +1243,155 @@ ] +class TestPositionalAfterOptionalOneOrMore(ParserTestCase): + """Tests specifying a positional that follows an arg with nargs=+""" + """http://bugs.python.org/file18328/test_pos_after_var_args.patch""" + + argument_signatures = [Sig('-x', nargs='+'), Sig('y')] + failures = ['', '-x'] + successes = [ + ('-x foo bar', NS(x=['foo',], y='bar')), + ('-x foo bar baz', NS(x=['foo', 'bar'], y='baz')), + ('-x foo bar baz buzz', NS(x=['foo', 'bar', 'baz'], y='buzz')), + ] + + +class TestPositionalsAfterOptionalsPlus(ParserTestCase): + """Tests specifying a positional that follows an arg with nargs=+ + http://bugs.python.org/issue9338#msg111270 + prototypical problem""" + + argument_signatures = [ + Sig('-w'), + Sig('-x', nargs='+'), + Sig('y', type=int), + Sig('z', nargs='*', type=int)] + failures = ['1 -x 2 3 -w 4 5 6' # error: unrecognized arguments: 5 6 + # z consumed in 1st argument group '1' + ] + successes = [ + # prototypical case for this issue + ('-w 1 -x 2 3 4 5', NS(w='1', x=['2', '3', '4'], y=5, z=[])), + ('-w 1 -x 2 3', NS(w='1', x=['2'], y=3, z=[])), + ('-w 1 -x 2 3 -w 4', NS(w='4', x=['2'], y=3, z=[]) ), + ('-x 1 2 -w 3', NS(w='3', x=['1'], y=2, z=[])), + ('-x 1 2 3 4 -w 5', NS(w='5', x=['1', '2', '3'], y=4, z=[])), + # strings that worked before + ('-w 1 -x2 3 4 5', NS(w='1', x=['2'], y=3, z=[4, 5])), + ('-w 1 2 -x 3 4 5', NS(w='1', x=['3', '4', '5'], y=2, z=[])), + ('-w 1 -x2 3', NS(w='1', x=['2'], y=3, z=[])), + ('-x 1 2 -w 3 4 5 6', NS(w='3', x=['1', '2'], y=4, z=[5, 6]) ), + ('-x 1 2 3 4 -w 5 6 7', NS(w='5', x=['1', '2', '3', '4'], y=6, z=[7])), + ('1 2 3 -x 4 5 -w 6', NS(w='6', x=['4', '5'], y=1, z=[2, 3])), + ('1 2 3', NS(w=None, x=None, y=1, z=[2, 3])), + ('1 -x 2 3 -w 4', NS(w='4', x=['2', '3'], y=1, z=[])), + ] + + +class TestPositionalArgAfterOptionalOptional(ParserTestCase): + """Tests specifying a positional that follows an arg with nargs=?""" + + argument_signatures = [ + Sig('-x', nargs='?'), + Sig('y', type=int)] + failures = [] + successes = [ + ('-x 1', NS(x=None, y=1)), + ('-x 1 2', NS(x='1', y=2)) + ] + + +class TestPositionalArgAfterNargsOptionalZeroOrMore(ParserTestCase): + """Tests specifying a positional that follows an arg with nargs=*""" + + argument_signatures = [ + Sig('-x', nargs='*'), + Sig('y', type=int)] + failures = [] + successes = [ + ('-x 1', NS(x=[], y=1)), + ('-x 1 2', NS(x=['1'], y=2)), + ('-x 1 2 3',NS(x=['1', '2'], y=3)) + ] + + +class TestPositional2AfterOptionalZeroOrMore(ParserTestCase): + """Tests specifying a positional that follows an arg with nargs=*""" + + argument_signatures = [ + Sig('-x', nargs='*'), + Sig('y', type=int, nargs=2)] + failures = ['-x 1'] + successes = [ + ('-x 1 2', NS(x=[], y=[1,2])), + ('-x 1 2 3', NS(x=['1'], y=[2,3])) + ] + + +class Test2PositionalsAfterOptionalsPlus(ParserTestCase): + """Tests specifying 2 positionals after optional with nargs=+""" + + argument_signatures = [ + Sig('-x', nargs='+'), + Sig('y', type=int, nargs=2), + Sig('z', type=int, nargs='+')] + failures = ['-x 1', '-x 1 2', '-x 1 2 3'] + successes = [ + ('-x 1 2 3 4', NS(x=['1'], y=[2,3], z=[4])), + ('-x 1 2 3 4 5', NS(x=['1','2'], y=[3,4], z=[5])) + ] + + +class Test2PositionalsAfterOptionalPlus(ParserTestCase): + """Tests specifying 2 positionals that follow an arg with nargs=+""" + + argument_signatures = [ + Sig('-w', nargs='+'), + Sig('-x', nargs='+'), + Sig('y', type=int,nargs=2), + Sig('z', type=int,nargs='+')] + failures = ['-x 1', '-x 1 2', '-x 1 2 3','-w 1 -x 2 3 4'] + successes = [ + ('-x 1 2 3 4', NS(w=None, x=['1'], y=[2, 3], z=[4])), + ('-x 1 2 3 4 5', NS(w=None, x=['1', '2'], y=[3, 4], z=[5])), + ('-w 1 -x 2 3 4 5', NS(w=['1'], x=['2'], y=[3, 4], z=[5])), + ('-w 1 2 -x 3 4 5 6', NS(w=['1', '2'], x=['3'], y=[4, 5], z=[6])), + ('-w 1 2 3 -x 4 5 6', NS(w=['1'], x=['4', '5'], y=[2, 3], z=[6])) + ] + + +class TestPositionalsAfterOptionalsComplex(ParserTestCase): + """Tests specifying positionals after optionals; complex case""" + """Something of a stress test""" + + argument_signatures = [ + Sig('-a', nargs='?'), + Sig('-b', nargs='+'), + Sig('-c', nargs='*'), + Sig('-j', action='store_true'), + Sig('-l'), + Sig('-m', nargs=1), + Sig('x', type=int), + Sig('y', type=int, nargs=2), + Sig('z', type=int, nargs='+')] + failures = [] + successes = [('1 2 3 4 5', NS(a=None, b=None, c=None, j=False, l=None, m=None, x=1, y=[2, 3], z=[4, 5])), + ('-a 1 2 3 4 5', NS(a='1', b=None, c=None, j=False, l=None, m=None, x=2, y=[3, 4], z=[5])), + ('-a 1 -b 2 3 4 5 6', NS(a='1', b=['2'], c=None, j=False, l=None, m=None, x=3, y=[4, 5], z=[6])), + ('-a 1 -b 2 3 -c 4 5 6 7 8', NS(a='1', b=['2', '3'], c=['4'], j=False, l=None, m=None, x=5, y=[6, 7], z=[8])), + ('-a 1 -b 2 3 -j -c 4 5 6 7 8', NS(a='1', b=['2', '3'], c=['4'], j=True, l=None, m=None, x=5, y=[6, 7], z=[8])), + ('-a 1 -l L -b 2 3 -c 4 5 6 7 8', NS(a='1', b=['2', '3'], c=['4'], j=False, l='L', m=None, x=5, y=[6, 7], z=[8])), + ('-a 1 -l L -b 2 3 -m M -c 4 5 6 7 8', NS(a='1', b=['2', '3'], c=['4'], j=False, l='L', m=['M'], x=5, y=[6, 7], z=[8])), + ('-a 1 -l L -c 2 3 -m M -b 4 5 6 7 8', NS(a='1', b=['4'], c=['2', '3'], j=False, l='L', m=['M'], x=5, y=[6, 7], z=[8])), + ('-l L -a 1 -j -b 2 3 -c 4 5 6 7 8', NS(a='1', b=['2', '3'], c=['4'], j=True, l='L', m=None, x=5, y=[6, 7], z=[8])), + ('-l L -a 1 -j -b 2 3 4 -m M -c 5', NS(a=None, b=['2'], c=[], j=True, l='L', m=['M'], x=1, y=[3, 4], z=[5])), + ('-l L -a 1 -j -b 2 3 4 -m M -c 5 6 7', NS(a='1', b=['2', '3'], c=[], j=True, l='L', m=['M'], x=4, y=[5, 6], z=[7])), + ('-lL -a1 -j -b 2 3 -c4 5 6 7 8', NS(a='1', b=['2', '3'], c=['4'], j=True, l='L', m=None, x=5, y=[6, 7], z=[8])), + ('-lL -a 1 -jb2 3 -c4 5 6 7', NS(a='1', b=['2'], c=['4'], j=True, l='L', m=None, x=3, y=[5, 6], z=[7])), + ('-lL -a 1 -jb 2 3 4 -c 5 6', NS(a=None, b=['2'], c=['5'], j=True, l='L', m=None, x=1, y=[3, 4], z=[6])), + ] + + class TestNargsRemainder(ParserTestCase): """Tests specifying a positional with nargs=REMAINDER"""