Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(132924)

Side by Side Diff: Lib/subprocess.py

Issue 23670: Modifications to support iOS as a development platform
Patch Set: Created 3 years, 8 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « Lib/site.py ('k') | Lib/sysconfig.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # subprocess - Subprocesses with accessible I/O streams 1 # subprocess - Subprocesses with accessible I/O streams
2 # 2 #
3 # For more information about this module, see PEP 324. 3 # For more information about this module, see PEP 324.
4 # 4 #
5 # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se> 5 # Copyright (c) 2003-2005 by Peter Astrand <astrand@lysator.liu.se>
6 # 6 #
7 # Licensed to PSF under a Contributor Agreement. 7 # Licensed to PSF under a Contributor Agreement.
8 # See http://www.python.org/2.4/license for licensing details. 8 # See http://www.python.org/2.4/license for licensing details.
9 9
10 r"""subprocess - Subprocesses with accessible I/O streams 10 r"""subprocess - Subprocesses with accessible I/O streams
(...skipping 438 matching lines...) Expand 10 before | Expand all | Expand 10 after
449 449
450 # poll/select have the advantage of not requiring any extra file 450 # poll/select have the advantage of not requiring any extra file
451 # descriptor, contrarily to epoll/kqueue (also, they require a single 451 # descriptor, contrarily to epoll/kqueue (also, they require a single
452 # syscall). 452 # syscall).
453 if hasattr(selectors, 'PollSelector'): 453 if hasattr(selectors, 'PollSelector'):
454 _PopenSelector = selectors.PollSelector 454 _PopenSelector = selectors.PollSelector
455 else: 455 else:
456 _PopenSelector = selectors.SelectSelector 456 _PopenSelector = selectors.SelectSelector
457 457
458 458
459 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput",
460 "getoutput", "check_output", "run", "CalledProcessError", "DEVNULL",
461 "SubprocessError", "TimeoutExpired", "CompletedProcess"]
462 # NOTE: We intentionally exclude list2cmdline as it is
463 # considered an internal implementation detail. issue10838.
464
465 if _mswindows:
466 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
467 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
468 STD_ERROR_HANDLE, SW_HIDE,
469 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
470
471 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
472 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
473 "STD_ERROR_HANDLE", "SW_HIDE",
474 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"])
475
476 class Handle(int):
477 closed = False
478
479 def Close(self, CloseHandle=_winapi.CloseHandle):
480 if not self.closed:
481 self.closed = True
482 CloseHandle(self)
483
484 def Detach(self):
485 if not self.closed:
486 self.closed = True
487 return int(self)
488 raise ValueError("already closed")
489
490 def __repr__(self):
491 return "%s(%d)" % (self.__class__.__name__, int(self))
492
493 __del__ = Close
494 __str__ = __repr__
495
496
497 # This lists holds Popen instances for which the underlying process had not
498 # exited at the time its __del__ method got called: those processes are wait()ed
499 # for synchronously from _cleanup() when a new Popen object is created, to avoid
500 # zombie processes.
501 _active = []
502
503 def _cleanup():
504 for inst in _active[:]:
505 res = inst._internal_poll(_deadstate=sys.maxsize)
506 if res is not None:
507 try:
508 _active.remove(inst)
509 except ValueError:
510 # This can happen if two threads create a new Popen instance.
511 # It's harmless that it was already removed, so ignore.
512 pass
513
514 PIPE = -1 459 PIPE = -1
515 STDOUT = -2 460 STDOUT = -2
516 DEVNULL = -3 461 DEVNULL = -3
517 462
518 463
519 # XXX This function is only used by multiprocessing and the test suite, 464 if sys.platform == 'ios':
520 # but it's here so that it can be imported when Python is compiled without 465 # iOS doesn't support subprocesses; however, some modules still import
521 # threads. 466 # subprocess, so it's helpful to retain the symbols in the namespace.
522 467 __all__ = ["PIPE", "STDOUT", "DEVNULL", "CalledProcessError",
523 def _optim_args_from_interpreter_flags(): 468 "SubprocessError", "TimeoutExpired"]
524 """Return a list of command-line arguments reproducing the current 469 else:
525 optimization settings in sys.flags.""" 470 __all__ = ["Popen", "PIPE", "STDOUT", "call", "check_call", "getstatusoutput ",
526 args = [] 471 "getoutput", "check_output", "run", "CalledProcessError", "DEVNUL L",
527 value = sys.flags.optimize 472 "SubprocessError", "TimeoutExpired", "CompletedProcess"]
528 if value > 0: 473 # NOTE: We intentionally exclude list2cmdline as it is
529 args.append('-' + 'O' * value) 474 # considered an internal implementation detail. issue10838.
530 return args 475
531 476 if _mswindows:
532 477 from _winapi import (CREATE_NEW_CONSOLE, CREATE_NEW_PROCESS_GROUP,
533 def _args_from_interpreter_flags(): 478 STD_INPUT_HANDLE, STD_OUTPUT_HANDLE,
534 """Return a list of command-line arguments reproducing the current 479 STD_ERROR_HANDLE, SW_HIDE,
535 settings in sys.flags and sys.warnoptions.""" 480 STARTF_USESTDHANDLES, STARTF_USESHOWWINDOW)
536 flag_opt_map = { 481
537 'debug': 'd', 482 __all__.extend(["CREATE_NEW_CONSOLE", "CREATE_NEW_PROCESS_GROUP",
538 # 'inspect': 'i', 483 "STD_INPUT_HANDLE", "STD_OUTPUT_HANDLE",
539 # 'interactive': 'i', 484 "STD_ERROR_HANDLE", "SW_HIDE",
540 'dont_write_bytecode': 'B', 485 "STARTF_USESTDHANDLES", "STARTF_USESHOWWINDOW"])
541 'no_user_site': 's', 486
542 'no_site': 'S', 487 class Handle(int):
543 'ignore_environment': 'E', 488 closed = False
544 'verbose': 'v', 489
545 'bytes_warning': 'b', 490 def Close(self, CloseHandle=_winapi.CloseHandle):
546 'quiet': 'q', 491 if not self.closed:
547 # -O is handled in _optim_args_from_interpreter_flags() 492 self.closed = True
548 } 493 CloseHandle(self)
549 args = _optim_args_from_interpreter_flags() 494
550 for flag, opt in flag_opt_map.items(): 495 def Detach(self):
551 v = getattr(sys.flags, flag) 496 if not self.closed:
552 if v > 0: 497 self.closed = True
553 args.append('-' + opt * v) 498 return int(self)
554 for opt in sys.warnoptions: 499 raise ValueError("already closed")
555 args.append('-W' + opt) 500
556 return args 501 def __repr__(self):
557 502 return "%s(%d)" % (self.__class__.__name__, int(self))
558 503
559 def call(*popenargs, timeout=None, **kwargs): 504 __del__ = Close
560 """Run command with arguments. Wait for command to complete or 505 __str__ = __repr__
561 timeout, then return the returncode attribute. 506
562 507
563 The arguments are the same as for the Popen constructor. Example: 508 # This lists holds Popen instances for which the underlying process had not
564 509 # exited at the time its __del__ method got called: those processes are wait ()ed
565 retcode = call(["ls", "-l"]) 510 # for synchronously from _cleanup() when a new Popen object is created, to a void
566 """ 511 # zombie processes.
567 with Popen(*popenargs, **kwargs) as p: 512 _active = []
513
514 def _cleanup():
515 for inst in _active[:]:
516 res = inst._internal_poll(_deadstate=sys.maxsize)
517 if res is not None:
518 try:
519 _active.remove(inst)
520 except ValueError:
521 # This can happen if two threads create a new Popen instance .
522 # It's harmless that it was already removed, so ignore.
523 pass
524
525
526 # XXX This function is only used by multiprocessing and the test suite,
527 # but it's here so that it can be imported when Python is compiled without
528 # threads.
529
530 def _optim_args_from_interpreter_flags():
531 """Return a list of command-line arguments reproducing the current
532 optimization settings in sys.flags."""
533 args = []
534 value = sys.flags.optimize
535 if value > 0:
536 args.append('-' + 'O' * value)
537 return args
538
539
540 def _args_from_interpreter_flags():
541 """Return a list of command-line arguments reproducing the current
542 settings in sys.flags and sys.warnoptions."""
543 flag_opt_map = {
544 'debug': 'd',
545 # 'inspect': 'i',
546 # 'interactive': 'i',
547 'dont_write_bytecode': 'B',
548 'no_user_site': 's',
549 'no_site': 'S',
550 'ignore_environment': 'E',
551 'verbose': 'v',
552 'bytes_warning': 'b',
553 'quiet': 'q',
554 # -O is handled in _optim_args_from_interpreter_flags()
555 }
556 args = _optim_args_from_interpreter_flags()
557 for flag, opt in flag_opt_map.items():
558 v = getattr(sys.flags, flag)
559 if v > 0:
560 args.append('-' + opt * v)
561 for opt in sys.warnoptions:
562 args.append('-W' + opt)
563 return args
564
565
566 def call(*popenargs, timeout=None, **kwargs):
567 """Run command with arguments. Wait for command to complete or
568 timeout, then return the returncode attribute.
569
570 The arguments are the same as for the Popen constructor. Example:
571
572 retcode = call(["ls", "-l"])
573 """
574 with Popen(*popenargs, **kwargs) as p:
575 try:
576 return p.wait(timeout=timeout)
577 except:
578 p.kill()
579 p.wait()
580 raise
581
582
583 def check_call(*popenargs, **kwargs):
584 """Run command with arguments. Wait for command to complete. If
585 the exit code was zero then return, otherwise raise
586 CalledProcessError. The CalledProcessError object will have the
587 return code in the returncode attribute.
588
589 The arguments are the same as for the call function. Example:
590
591 check_call(["ls", "-l"])
592 """
593 retcode = call(*popenargs, **kwargs)
594 if retcode:
595 cmd = kwargs.get("args")
596 if cmd is None:
597 cmd = popenargs[0]
598 raise CalledProcessError(retcode, cmd)
599 return 0
600
601
602 def check_output(*popenargs, timeout=None, **kwargs):
603 r"""Run command with arguments and return its output.
604
605 If the exit code was non-zero it raises a CalledProcessError. The
606 CalledProcessError object will have the return code in the returncode
607 attribute and output in the output attribute.
608
609 The arguments are the same as for the Popen constructor. Example:
610
611 >>> check_output(["ls", "-l", "/dev/null"])
612 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n'
613
614 The stdout argument is not allowed as it is used internally.
615 To capture standard error in the result, use stderr=STDOUT.
616
617 >>> check_output(["/bin/sh", "-c",
618 ... "ls -l non_existent_file ; exit 0"],
619 ... stderr=STDOUT)
620 b'ls: non_existent_file: No such file or directory\n'
621
622 There is an additional optional argument, "input", allowing you to
623 pass a string to the subprocess's stdin. If you use this argument
624 you may not also use the Popen constructor's "stdin" argument, as
625 it too will be used internally. Example:
626
627 >>> check_output(["sed", "-e", "s/foo/bar/"],
628 ... input=b"when in the course of fooman events\n")
629 b'when in the course of barman events\n'
630
631 If universal_newlines=True is passed, the "input" argument must be a
632 string and the return value will be a string rather than bytes.
633 """
634 if 'stdout' in kwargs:
635 raise ValueError('stdout argument not allowed, it will be overridden .')
636
637 if 'input' in kwargs and kwargs['input'] is None:
638 # Explicitly passing input=None was previously equivalent to passing an
639 # empty string. That is maintained here for backwards compatibility.
640 kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
641
642 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
643 **kwargs).stdout
644
645
646 class CompletedProcess(object):
647 """A process that has finished running.
648
649 This is returned by run().
650
651 Attributes:
652 args: The list or str args passed to run().
653 returncode: The exit code of the process, negative for signals.
654 stdout: The standard output (None if not captured).
655 stderr: The standard error (None if not captured).
656 """
657 def __init__(self, args, returncode, stdout=None, stderr=None):
658 self.args = args
659 self.returncode = returncode
660 self.stdout = stdout
661 self.stderr = stderr
662
663 def __repr__(self):
664 args = ['args={!r}'.format(self.args),
665 'returncode={!r}'.format(self.returncode)]
666 if self.stdout is not None:
667 args.append('stdout={!r}'.format(self.stdout))
668 if self.stderr is not None:
669 args.append('stderr={!r}'.format(self.stderr))
670 return "{}({})".format(type(self).__name__, ', '.join(args))
671
672 def check_returncode(self):
673 """Raise CalledProcessError if the exit code is non-zero."""
674 if self.returncode:
675 raise CalledProcessError(self.returncode, self.args, self.stdout ,
676 self.stderr)
677
678
679 def run(*popenargs, input=None, timeout=None, check=False, **kwargs):
680 """Run command with arguments and return a CompletedProcess instance.
681
682 The returned instance will have attributes args, returncode, stdout and
683 stderr. By default, stdout and stderr are not captured, and those attrib utes
684 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture th em.
685
686 If check is True and the exit code was non-zero, it raises a
687 CalledProcessError. The CalledProcessError object will have the return c ode
688 in the returncode attribute, and output & stderr attributes if those str eams
689 were captured.
690
691 If timeout is given, and the process takes too long, a TimeoutExpired
692 exception will be raised.
693
694 There is an optional argument "input", allowing you to
695 pass a string to the subprocess's stdin. If you use this argument
696 you may not also use the Popen constructor's "stdin" argument, as
697 it will be used internally.
698
699 The other arguments are the same as for the Popen constructor.
700
701 If universal_newlines=True is passed, the "input" argument must be a
702 string and stdout/stderr in the returned object will be strings rather t han
703 bytes.
704 """
705 if input is not None:
706 if 'stdin' in kwargs:
707 raise ValueError('stdin and input arguments may not both be used .')
708 kwargs['stdin'] = PIPE
709
710 with Popen(*popenargs, **kwargs) as process:
711 try:
712 stdout, stderr = process.communicate(input, timeout=timeout)
713 except TimeoutExpired:
714 process.kill()
715 stdout, stderr = process.communicate()
716 raise TimeoutExpired(process.args, timeout, output=stdout,
717 stderr=stderr)
718 except:
719 process.kill()
720 process.wait()
721 raise
722 retcode = process.poll()
723 if check and retcode:
724 raise CalledProcessError(retcode, process.args,
725 output=stdout, stderr=stderr)
726 return CompletedProcess(process.args, retcode, stdout, stderr)
727
728
729 def list2cmdline(seq):
730 """
731 Translate a sequence of arguments into a command line
732 string, using the same rules as the MS C runtime:
733
734 1) Arguments are delimited by white space, which is either a
735 space or a tab.
736
737 2) A string surrounded by double quotation marks is
738 interpreted as a single argument, regardless of white space
739 contained within. A quoted string can be embedded in an
740 argument.
741
742 3) A double quotation mark preceded by a backslash is
743 interpreted as a literal double quotation mark.
744
745 4) Backslashes are interpreted literally, unless they
746 immediately precede a double quotation mark.
747
748 5) If backslashes immediately precede a double quotation mark,
749 every pair of backslashes is interpreted as a literal
750 backslash. If the number of backslashes is odd, the last
751 backslash escapes the next double quotation mark as
752 described in rule 3.
753 """
754
755 # See
756 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
757 # or search http://msdn.microsoft.com for
758 # "Parsing C++ Command-Line Arguments"
759 result = []
760 needquote = False
761 for arg in seq:
762 bs_buf = []
763
764 # Add a space to separate this argument from the others
765 if result:
766 result.append(' ')
767
768 needquote = (" " in arg) or ("\t" in arg) or not arg
769 if needquote:
770 result.append('"')
771
772 for c in arg:
773 if c == '\\':
774 # Don't know if we need to double yet.
775 bs_buf.append(c)
776 elif c == '"':
777 # Double backslashes.
778 result.append('\\' * len(bs_buf)*2)
779 bs_buf = []
780 result.append('\\"')
781 else:
782 # Normal char
783 if bs_buf:
784 result.extend(bs_buf)
785 bs_buf = []
786 result.append(c)
787
788 # Add remaining backslashes, if any.
789 if bs_buf:
790 result.extend(bs_buf)
791
792 if needquote:
793 result.extend(bs_buf)
794 result.append('"')
795
796 return ''.join(result)
797
798
799 # Various tools for executing commands and looking at their output and statu s.
800 #
801
802 def getstatusoutput(cmd):
803 """ Return (status, output) of executing cmd in a shell.
804
805 Execute the string 'cmd' in a shell with 'check_output' and
806 return a 2-tuple (status, output). Universal newlines mode is used,
807 meaning that the result with be decoded to a string.
808
809 A trailing newline is stripped from the output.
810 The exit status for the command can be interpreted
811 according to the rules for the function 'wait'. Example:
812
813 >>> import subprocess
814 >>> subprocess.getstatusoutput('ls /bin/ls')
815 (0, '/bin/ls')
816 >>> subprocess.getstatusoutput('cat /bin/junk')
817 (256, 'cat: /bin/junk: No such file or directory')
818 >>> subprocess.getstatusoutput('/bin/junk')
819 (256, 'sh: /bin/junk: not found')
820 """
568 try: 821 try:
569 return p.wait(timeout=timeout) 822 data = check_output(cmd, shell=True, universal_newlines=True, stderr =STDOUT)
570 except: 823 status = 0
571 p.kill() 824 except CalledProcessError as ex:
572 p.wait() 825 data = ex.output
573 raise 826 status = ex.returncode
574 827 if data[-1:] == '\n':
575 828 data = data[:-1]
576 def check_call(*popenargs, **kwargs): 829 return status, data
577 """Run command with arguments. Wait for command to complete. If 830
578 the exit code was zero then return, otherwise raise 831 def getoutput(cmd):
579 CalledProcessError. The CalledProcessError object will have the 832 """Return output (stdout or stderr) of executing cmd in a shell.
580 return code in the returncode attribute. 833
581 834 Like getstatusoutput(), except the exit status is ignored and the return
582 The arguments are the same as for the call function. Example: 835 value is a string containing the command's output. Example:
583 836
584 check_call(["ls", "-l"]) 837 >>> import subprocess
585 """ 838 >>> subprocess.getoutput('ls /bin/ls')
586 retcode = call(*popenargs, **kwargs) 839 '/bin/ls'
587 if retcode: 840 """
588 cmd = kwargs.get("args") 841 return getstatusoutput(cmd)[1]
589 if cmd is None: 842
590 cmd = popenargs[0] 843
591 raise CalledProcessError(retcode, cmd) 844 _PLATFORM_DEFAULT_CLOSE_FDS = object()
592 return 0 845
593 846
594 847 class Popen(object):
595 def check_output(*popenargs, timeout=None, **kwargs): 848
596 r"""Run command with arguments and return its output. 849 _child_created = False # Set here since __del__ checks it
597 850
598 If the exit code was non-zero it raises a CalledProcessError. The 851 def __init__(self, args, bufsize=-1, executable=None,
599 CalledProcessError object will have the return code in the returncode 852 stdin=None, stdout=None, stderr=None,
600 attribute and output in the output attribute. 853 preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS,
601 854 shell=False, cwd=None, env=None, universal_newlines=False,
602 The arguments are the same as for the Popen constructor. Example: 855 startupinfo=None, creationflags=0,
603 856 restore_signals=True, start_new_session=False,
604 >>> check_output(["ls", "-l", "/dev/null"]) 857 pass_fds=()):
605 b'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' 858 """Create new Popen instance."""
606 859 _cleanup()
607 The stdout argument is not allowed as it is used internally. 860 # Held while anything is calling waitpid before returncode has been
608 To capture standard error in the result, use stderr=STDOUT. 861 # updated to prevent clobbering returncode if wait() or poll() are
609 862 # called from multiple threads at once. After acquiring the lock,
610 >>> check_output(["/bin/sh", "-c", 863 # code must re-check self.returncode to see if another thread just
611 ... "ls -l non_existent_file ; exit 0"], 864 # finished a waitpid() call.
612 ... stderr=STDOUT) 865 self._waitpid_lock = threading.Lock()
613 b'ls: non_existent_file: No such file or directory\n' 866
614 867 self._input = None
615 There is an additional optional argument, "input", allowing you to 868 self._communication_started = False
616 pass a string to the subprocess's stdin. If you use this argument 869 if bufsize is None:
617 you may not also use the Popen constructor's "stdin" argument, as 870 bufsize = -1 # Restore default
618 it too will be used internally. Example: 871 if not isinstance(bufsize, int):
619 872 raise TypeError("bufsize must be an integer")
620 >>> check_output(["sed", "-e", "s/foo/bar/"], 873
621 ... input=b"when in the course of fooman events\n") 874 if _mswindows:
622 b'when in the course of barman events\n' 875 if preexec_fn is not None:
623 876 raise ValueError("preexec_fn is not supported on Windows "
624 If universal_newlines=True is passed, the "input" argument must be a 877 "platforms")
625 string and the return value will be a string rather than bytes. 878 any_stdio_set = (stdin is not None or stdout is not None or
626 """ 879 stderr is not None)
627 if 'stdout' in kwargs: 880 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
628 raise ValueError('stdout argument not allowed, it will be overridden.') 881 if any_stdio_set:
629 882 close_fds = False
630 if 'input' in kwargs and kwargs['input'] is None: 883 else:
631 # Explicitly passing input=None was previously equivalent to passing an 884 close_fds = True
632 # empty string. That is maintained here for backwards compatibility. 885 elif close_fds and any_stdio_set:
633 kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b'' 886 raise ValueError(
634 887 "close_fds is not supported on Windows platforms"
635 return run(*popenargs, stdout=PIPE, timeout=timeout, check=True, 888 " if you redirect stdin/stdout/stderr")
636 **kwargs).stdout
637
638
639 class CompletedProcess(object):
640 """A process that has finished running.
641
642 This is returned by run().
643
644 Attributes:
645 args: The list or str args passed to run().
646 returncode: The exit code of the process, negative for signals.
647 stdout: The standard output (None if not captured).
648 stderr: The standard error (None if not captured).
649 """
650 def __init__(self, args, returncode, stdout=None, stderr=None):
651 self.args = args
652 self.returncode = returncode
653 self.stdout = stdout
654 self.stderr = stderr
655
656 def __repr__(self):
657 args = ['args={!r}'.format(self.args),
658 'returncode={!r}'.format(self.returncode)]
659 if self.stdout is not None:
660 args.append('stdout={!r}'.format(self.stdout))
661 if self.stderr is not None:
662 args.append('stderr={!r}'.format(self.stderr))
663 return "{}({})".format(type(self).__name__, ', '.join(args))
664
665 def check_returncode(self):
666 """Raise CalledProcessError if the exit code is non-zero."""
667 if self.returncode:
668 raise CalledProcessError(self.returncode, self.args, self.stdout,
669 self.stderr)
670
671
672 def run(*popenargs, input=None, timeout=None, check=False, **kwargs):
673 """Run command with arguments and return a CompletedProcess instance.
674
675 The returned instance will have attributes args, returncode, stdout and
676 stderr. By default, stdout and stderr are not captured, and those attributes
677 will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.
678
679 If check is True and the exit code was non-zero, it raises a
680 CalledProcessError. The CalledProcessError object will have the return code
681 in the returncode attribute, and output & stderr attributes if those streams
682 were captured.
683
684 If timeout is given, and the process takes too long, a TimeoutExpired
685 exception will be raised.
686
687 There is an optional argument "input", allowing you to
688 pass a string to the subprocess's stdin. If you use this argument
689 you may not also use the Popen constructor's "stdin" argument, as
690 it will be used internally.
691
692 The other arguments are the same as for the Popen constructor.
693
694 If universal_newlines=True is passed, the "input" argument must be a
695 string and stdout/stderr in the returned object will be strings rather than
696 bytes.
697 """
698 if input is not None:
699 if 'stdin' in kwargs:
700 raise ValueError('stdin and input arguments may not both be used.')
701 kwargs['stdin'] = PIPE
702
703 with Popen(*popenargs, **kwargs) as process:
704 try:
705 stdout, stderr = process.communicate(input, timeout=timeout)
706 except TimeoutExpired:
707 process.kill()
708 stdout, stderr = process.communicate()
709 raise TimeoutExpired(process.args, timeout, output=stdout,
710 stderr=stderr)
711 except:
712 process.kill()
713 process.wait()
714 raise
715 retcode = process.poll()
716 if check and retcode:
717 raise CalledProcessError(retcode, process.args,
718 output=stdout, stderr=stderr)
719 return CompletedProcess(process.args, retcode, stdout, stderr)
720
721
722 def list2cmdline(seq):
723 """
724 Translate a sequence of arguments into a command line
725 string, using the same rules as the MS C runtime:
726
727 1) Arguments are delimited by white space, which is either a
728 space or a tab.
729
730 2) A string surrounded by double quotation marks is
731 interpreted as a single argument, regardless of white space
732 contained within. A quoted string can be embedded in an
733 argument.
734
735 3) A double quotation mark preceded by a backslash is
736 interpreted as a literal double quotation mark.
737
738 4) Backslashes are interpreted literally, unless they
739 immediately precede a double quotation mark.
740
741 5) If backslashes immediately precede a double quotation mark,
742 every pair of backslashes is interpreted as a literal
743 backslash. If the number of backslashes is odd, the last
744 backslash escapes the next double quotation mark as
745 described in rule 3.
746 """
747
748 # See
749 # http://msdn.microsoft.com/en-us/library/17w5ykft.aspx
750 # or search http://msdn.microsoft.com for
751 # "Parsing C++ Command-Line Arguments"
752 result = []
753 needquote = False
754 for arg in seq:
755 bs_buf = []
756
757 # Add a space to separate this argument from the others
758 if result:
759 result.append(' ')
760
761 needquote = (" " in arg) or ("\t" in arg) or not arg
762 if needquote:
763 result.append('"')
764
765 for c in arg:
766 if c == '\\':
767 # Don't know if we need to double yet.
768 bs_buf.append(c)
769 elif c == '"':
770 # Double backslashes.
771 result.append('\\' * len(bs_buf)*2)
772 bs_buf = []
773 result.append('\\"')
774 else: 889 else:
775 # Normal char 890 # POSIX
776 if bs_buf: 891 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
777 result.extend(bs_buf)
778 bs_buf = []
779 result.append(c)
780
781 # Add remaining backslashes, if any.
782 if bs_buf:
783 result.extend(bs_buf)
784
785 if needquote:
786 result.extend(bs_buf)
787 result.append('"')
788
789 return ''.join(result)
790
791
792 # Various tools for executing commands and looking at their output and status.
793 #
794
795 def getstatusoutput(cmd):
796 """ Return (status, output) of executing cmd in a shell.
797
798 Execute the string 'cmd' in a shell with 'check_output' and
799 return a 2-tuple (status, output). Universal newlines mode is used,
800 meaning that the result with be decoded to a string.
801
802 A trailing newline is stripped from the output.
803 The exit status for the command can be interpreted
804 according to the rules for the function 'wait'. Example:
805
806 >>> import subprocess
807 >>> subprocess.getstatusoutput('ls /bin/ls')
808 (0, '/bin/ls')
809 >>> subprocess.getstatusoutput('cat /bin/junk')
810 (256, 'cat: /bin/junk: No such file or directory')
811 >>> subprocess.getstatusoutput('/bin/junk')
812 (256, 'sh: /bin/junk: not found')
813 """
814 try:
815 data = check_output(cmd, shell=True, universal_newlines=True, stderr=STD OUT)
816 status = 0
817 except CalledProcessError as ex:
818 data = ex.output
819 status = ex.returncode
820 if data[-1:] == '\n':
821 data = data[:-1]
822 return status, data
823
824 def getoutput(cmd):
825 """Return output (stdout or stderr) of executing cmd in a shell.
826
827 Like getstatusoutput(), except the exit status is ignored and the return
828 value is a string containing the command's output. Example:
829
830 >>> import subprocess
831 >>> subprocess.getoutput('ls /bin/ls')
832 '/bin/ls'
833 """
834 return getstatusoutput(cmd)[1]
835
836
837 _PLATFORM_DEFAULT_CLOSE_FDS = object()
838
839
840 class Popen(object):
841
842 _child_created = False # Set here since __del__ checks it
843
844 def __init__(self, args, bufsize=-1, executable=None,
845 stdin=None, stdout=None, stderr=None,
846 preexec_fn=None, close_fds=_PLATFORM_DEFAULT_CLOSE_FDS,
847 shell=False, cwd=None, env=None, universal_newlines=False,
848 startupinfo=None, creationflags=0,
849 restore_signals=True, start_new_session=False,
850 pass_fds=()):
851 """Create new Popen instance."""
852 _cleanup()
853 # Held while anything is calling waitpid before returncode has been
854 # updated to prevent clobbering returncode if wait() or poll() are
855 # called from multiple threads at once. After acquiring the lock,
856 # code must re-check self.returncode to see if another thread just
857 # finished a waitpid() call.
858 self._waitpid_lock = threading.Lock()
859
860 self._input = None
861 self._communication_started = False
862 if bufsize is None:
863 bufsize = -1 # Restore default
864 if not isinstance(bufsize, int):
865 raise TypeError("bufsize must be an integer")
866
867 if _mswindows:
868 if preexec_fn is not None:
869 raise ValueError("preexec_fn is not supported on Windows "
870 "platforms")
871 any_stdio_set = (stdin is not None or stdout is not None or
872 stderr is not None)
873 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS:
874 if any_stdio_set:
875 close_fds = False
876 else:
877 close_fds = True 892 close_fds = True
878 elif close_fds and any_stdio_set: 893 if pass_fds and not close_fds:
879 raise ValueError( 894 warnings.warn("pass_fds overriding close_fds.", RuntimeWarni ng)
880 "close_fds is not supported on Windows platforms" 895 close_fds = True
881 " if you redirect stdin/stdout/stderr") 896 if startupinfo is not None:
882 else: 897 raise ValueError("startupinfo is only supported on Windows "
883 # POSIX 898 "platforms")
884 if close_fds is _PLATFORM_DEFAULT_CLOSE_FDS: 899 if creationflags != 0:
885 close_fds = True 900 raise ValueError("creationflags is only supported on Windows "
886 if pass_fds and not close_fds: 901 "platforms")
887 warnings.warn("pass_fds overriding close_fds.", RuntimeWarning) 902
888 close_fds = True 903 self.args = args
889 if startupinfo is not None: 904 self.stdin = None
890 raise ValueError("startupinfo is only supported on Windows " 905 self.stdout = None
891 "platforms") 906 self.stderr = None
892 if creationflags != 0: 907 self.pid = None
893 raise ValueError("creationflags is only supported on Windows " 908 self.returncode = None
894 "platforms") 909 self.universal_newlines = universal_newlines
895 910
896 self.args = args 911 # Input and output objects. The general principle is like
897 self.stdin = None 912 # this:
898 self.stdout = None 913 #
899 self.stderr = None 914 # Parent Child
900 self.pid = None 915 # ------ -----
901 self.returncode = None 916 # p2cwrite ---stdin---> p2cread
902 self.universal_newlines = universal_newlines 917 # c2pread <--stdout--- c2pwrite
903 918 # errread <--stderr--- errwrite
904 # Input and output objects. The general principle is like 919 #
905 # this: 920 # On POSIX, the child objects are file descriptors. On
906 # 921 # Windows, these are Windows file handles. The parent objects
907 # Parent Child 922 # are file descriptors on both platforms. The parent objects
908 # ------ ----- 923 # are -1 when not using PIPEs. The child objects are -1
909 # p2cwrite ---stdin---> p2cread 924 # when not redirecting.
910 # c2pread <--stdout--- c2pwrite 925
911 # errread <--stderr--- errwrite 926 (p2cread, p2cwrite,
912 # 927 c2pread, c2pwrite,
913 # On POSIX, the child objects are file descriptors. On 928 errread, errwrite) = self._get_handles(stdin, stdout, stderr)
914 # Windows, these are Windows file handles. The parent objects 929
915 # are file descriptors on both platforms. The parent objects 930 # We wrap OS handles *before* launching the child, otherwise a
916 # are -1 when not using PIPEs. The child objects are -1 931 # quickly terminating child could make our fds unwrappable
917 # when not redirecting. 932 # (see #8458).
918 933
919 (p2cread, p2cwrite, 934 if _mswindows:
920 c2pread, c2pwrite, 935 if p2cwrite != -1:
921 errread, errwrite) = self._get_handles(stdin, stdout, stderr) 936 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
922 937 if c2pread != -1:
923 # We wrap OS handles *before* launching the child, otherwise a 938 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
924 # quickly terminating child could make our fds unwrappable 939 if errread != -1:
925 # (see #8458). 940 errread = msvcrt.open_osfhandle(errread.Detach(), 0)
926 941
927 if _mswindows:
928 if p2cwrite != -1: 942 if p2cwrite != -1:
929 p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0) 943 self.stdin = io.open(p2cwrite, 'wb', bufsize)
944 if universal_newlines:
945 self.stdin = io.TextIOWrapper(self.stdin, write_through=True ,
946 line_buffering=(bufsize == 1))
930 if c2pread != -1: 947 if c2pread != -1:
931 c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0) 948 self.stdout = io.open(c2pread, 'rb', bufsize)
949 if universal_newlines:
950 self.stdout = io.TextIOWrapper(self.stdout)
932 if errread != -1: 951 if errread != -1:
933 errread = msvcrt.open_osfhandle(errread.Detach(), 0) 952 self.stderr = io.open(errread, 'rb', bufsize)
934 953 if universal_newlines:
935 if p2cwrite != -1: 954 self.stderr = io.TextIOWrapper(self.stderr)
936 self.stdin = io.open(p2cwrite, 'wb', bufsize) 955
937 if universal_newlines: 956 self._closed_child_pipe_fds = False
938 self.stdin = io.TextIOWrapper(self.stdin, write_through=True, 957 try:
939 line_buffering=(bufsize == 1)) 958 self._execute_child(args, executable, preexec_fn, close_fds,
940 if c2pread != -1: 959 pass_fds, cwd, env,
941 self.stdout = io.open(c2pread, 'rb', bufsize) 960 startupinfo, creationflags, shell,
942 if universal_newlines: 961 p2cread, p2cwrite,
943 self.stdout = io.TextIOWrapper(self.stdout) 962 c2pread, c2pwrite,
944 if errread != -1: 963 errread, errwrite,
945 self.stderr = io.open(errread, 'rb', bufsize) 964 restore_signals, start_new_session)
946 if universal_newlines: 965 except:
947 self.stderr = io.TextIOWrapper(self.stderr) 966 # Cleanup if the child failed starting.
948 967 for f in filter(None, (self.stdin, self.stdout, self.stderr)):
949 self._closed_child_pipe_fds = False
950 try:
951 self._execute_child(args, executable, preexec_fn, close_fds,
952 pass_fds, cwd, env,
953 startupinfo, creationflags, shell,
954 p2cread, p2cwrite,
955 c2pread, c2pwrite,
956 errread, errwrite,
957 restore_signals, start_new_session)
958 except:
959 # Cleanup if the child failed starting.
960 for f in filter(None, (self.stdin, self.stdout, self.stderr)):
961 try:
962 f.close()
963 except OSError:
964 pass # Ignore EBADF or other errors.
965
966 if not self._closed_child_pipe_fds:
967 to_close = []
968 if stdin == PIPE:
969 to_close.append(p2cread)
970 if stdout == PIPE:
971 to_close.append(c2pwrite)
972 if stderr == PIPE:
973 to_close.append(errwrite)
974 if hasattr(self, '_devnull'):
975 to_close.append(self._devnull)
976 for fd in to_close:
977 try: 968 try:
978 os.close(fd) 969 f.close()
979 except OSError: 970 except OSError:
980 pass 971 pass # Ignore EBADF or other errors.
981 972
982 raise 973 if not self._closed_child_pipe_fds:
983 974 to_close = []
984 975 if stdin == PIPE:
985 def _translate_newlines(self, data, encoding): 976 to_close.append(p2cread)
986 data = data.decode(encoding) 977 if stdout == PIPE:
987 return data.replace("\r\n", "\n").replace("\r", "\n") 978 to_close.append(c2pwrite)
988 979 if stderr == PIPE:
989 def __enter__(self): 980 to_close.append(errwrite)
990 return self 981 if hasattr(self, '_devnull'):
991 982 to_close.append(self._devnull)
992 def __exit__(self, type, value, traceback): 983 for fd in to_close:
993 if self.stdout: 984 try:
994 self.stdout.close() 985 os.close(fd)
995 if self.stderr: 986 except OSError:
996 self.stderr.close() 987 pass
997 try: # Flushing a BufferedWriter may raise an error 988
998 if self.stdin: 989 raise
999 self.stdin.close() 990
1000 finally: 991
1001 # Wait for the process to terminate, to avoid zombies. 992 def _translate_newlines(self, data, encoding):
1002 self.wait() 993 data = data.decode(encoding)
1003 994 return data.replace("\r\n", "\n").replace("\r", "\n")
1004 def __del__(self, _maxsize=sys.maxsize): 995
1005 if not self._child_created: 996 def __enter__(self):
1006 # We didn't get to successfully create a child process. 997 return self
1007 return 998
1008 # In case the child hasn't been waited on, check if it's done. 999 def __exit__(self, type, value, traceback):
1009 self._internal_poll(_deadstate=_maxsize)
1010 if self.returncode is None and _active is not None:
1011 # Child is still running, keep us alive until we can wait on it.
1012 _active.append(self)
1013
1014 def _get_devnull(self):
1015 if not hasattr(self, '_devnull'):
1016 self._devnull = os.open(os.devnull, os.O_RDWR)
1017 return self._devnull
1018
1019 def _stdin_write(self, input):
1020 if input:
1021 try:
1022 self.stdin.write(input)
1023 except BrokenPipeError:
1024 # communicate() must ignore broken pipe error
1025 pass
1026 except OSError as e:
1027 if e.errno == errno.EINVAL and self.poll() is not None:
1028 # Issue #19612: On Windows, stdin.write() fails with EINVAL
1029 # if the process already exited before the write
1030 pass
1031 else:
1032 raise
1033 self.stdin.close()
1034
1035 def communicate(self, input=None, timeout=None):
1036 """Interact with process: Send data to stdin. Read data from
1037 stdout and stderr, until end-of-file is reached. Wait for
1038 process to terminate.
1039
1040 The optional "input" argument should be data to be sent to the
1041 child process (if self.universal_newlines is True, this should
1042 be a string; if it is False, "input" should be bytes), or
1043 None, if no data should be sent to the child.
1044
1045 communicate() returns a tuple (stdout, stderr). These will be
1046 bytes or, if self.universal_newlines was True, a string.
1047 """
1048
1049 if self._communication_started and input:
1050 raise ValueError("Cannot send input after starting communication")
1051
1052 # Optimization: If we are not worried about timeouts, we haven't
1053 # started communicating, and we have one or zero pipes, using select()
1054 # or threads is unnecessary.
1055 if (timeout is None and not self._communication_started and
1056 [self.stdin, self.stdout, self.stderr].count(None) >= 2):
1057 stdout = None
1058 stderr = None
1059 if self.stdin:
1060 self._stdin_write(input)
1061 elif self.stdout:
1062 stdout = self.stdout.read()
1063 self.stdout.close()
1064 elif self.stderr:
1065 stderr = self.stderr.read()
1066 self.stderr.close()
1067 self.wait()
1068 else:
1069 if timeout is not None:
1070 endtime = _time() + timeout
1071 else:
1072 endtime = None
1073
1074 try:
1075 stdout, stderr = self._communicate(input, endtime, timeout)
1076 finally:
1077 self._communication_started = True
1078
1079 sts = self.wait(timeout=self._remaining_time(endtime))
1080
1081 return (stdout, stderr)
1082
1083
1084 def poll(self):
1085 return self._internal_poll()
1086
1087
1088 def _remaining_time(self, endtime):
1089 """Convenience for _communicate when computing timeouts."""
1090 if endtime is None:
1091 return None
1092 else:
1093 return endtime - _time()
1094
1095
1096 def _check_timeout(self, endtime, orig_timeout):
1097 """Convenience for checking if a timeout has expired."""
1098 if endtime is None:
1099 return
1100 if _time() > endtime:
1101 raise TimeoutExpired(self.args, orig_timeout)
1102
1103
1104 if _mswindows:
1105 #
1106 # Windows methods
1107 #
1108 def _get_handles(self, stdin, stdout, stderr):
1109 """Construct and return tuple with IO objects:
1110 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1111 """
1112 if stdin is None and stdout is None and stderr is None:
1113 return (-1, -1, -1, -1, -1, -1)
1114
1115 p2cread, p2cwrite = -1, -1
1116 c2pread, c2pwrite = -1, -1
1117 errread, errwrite = -1, -1
1118
1119 if stdin is None:
1120 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
1121 if p2cread is None:
1122 p2cread, _ = _winapi.CreatePipe(None, 0)
1123 p2cread = Handle(p2cread)
1124 _winapi.CloseHandle(_)
1125 elif stdin == PIPE:
1126 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
1127 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
1128 elif stdin == DEVNULL:
1129 p2cread = msvcrt.get_osfhandle(self._get_devnull())
1130 elif isinstance(stdin, int):
1131 p2cread = msvcrt.get_osfhandle(stdin)
1132 else:
1133 # Assuming file-like object
1134 p2cread = msvcrt.get_osfhandle(stdin.fileno())
1135 p2cread = self._make_inheritable(p2cread)
1136
1137 if stdout is None:
1138 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
1139 if c2pwrite is None:
1140 _, c2pwrite = _winapi.CreatePipe(None, 0)
1141 c2pwrite = Handle(c2pwrite)
1142 _winapi.CloseHandle(_)
1143 elif stdout == PIPE:
1144 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
1145 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
1146 elif stdout == DEVNULL:
1147 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
1148 elif isinstance(stdout, int):
1149 c2pwrite = msvcrt.get_osfhandle(stdout)
1150 else:
1151 # Assuming file-like object
1152 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
1153 c2pwrite = self._make_inheritable(c2pwrite)
1154
1155 if stderr is None:
1156 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
1157 if errwrite is None:
1158 _, errwrite = _winapi.CreatePipe(None, 0)
1159 errwrite = Handle(errwrite)
1160 _winapi.CloseHandle(_)
1161 elif stderr == PIPE:
1162 errread, errwrite = _winapi.CreatePipe(None, 0)
1163 errread, errwrite = Handle(errread), Handle(errwrite)
1164 elif stderr == STDOUT:
1165 errwrite = c2pwrite
1166 elif stderr == DEVNULL:
1167 errwrite = msvcrt.get_osfhandle(self._get_devnull())
1168 elif isinstance(stderr, int):
1169 errwrite = msvcrt.get_osfhandle(stderr)
1170 else:
1171 # Assuming file-like object
1172 errwrite = msvcrt.get_osfhandle(stderr.fileno())
1173 errwrite = self._make_inheritable(errwrite)
1174
1175 return (p2cread, p2cwrite,
1176 c2pread, c2pwrite,
1177 errread, errwrite)
1178
1179
1180 def _make_inheritable(self, handle):
1181 """Return a duplicate of handle, which is inheritable"""
1182 h = _winapi.DuplicateHandle(
1183 _winapi.GetCurrentProcess(), handle,
1184 _winapi.GetCurrentProcess(), 0, 1,
1185 _winapi.DUPLICATE_SAME_ACCESS)
1186 return Handle(h)
1187
1188
1189 def _execute_child(self, args, executable, preexec_fn, close_fds,
1190 pass_fds, cwd, env,
1191 startupinfo, creationflags, shell,
1192 p2cread, p2cwrite,
1193 c2pread, c2pwrite,
1194 errread, errwrite,
1195 unused_restore_signals, unused_start_new_session):
1196 """Execute program (MS Windows version)"""
1197
1198 assert not pass_fds, "pass_fds not supported on Windows."
1199
1200 if not isinstance(args, str):
1201 args = list2cmdline(args)
1202
1203 # Process startup details
1204 if startupinfo is None:
1205 startupinfo = STARTUPINFO()
1206 if -1 not in (p2cread, c2pwrite, errwrite):
1207 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
1208 startupinfo.hStdInput = p2cread
1209 startupinfo.hStdOutput = c2pwrite
1210 startupinfo.hStdError = errwrite
1211
1212 if shell:
1213 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
1214 startupinfo.wShowWindow = _winapi.SW_HIDE
1215 comspec = os.environ.get("COMSPEC", "cmd.exe")
1216 args = '{} /c "{}"'.format (comspec, args)
1217
1218 # Start the process
1219 try:
1220 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
1221 # no special security
1222 None, None,
1223 int(not close_fds),
1224 creationflags,
1225 env,
1226 cwd,
1227 startupinfo)
1228 finally:
1229 # Child is launched. Close the parent's copy of those pipe
1230 # handles that only the child should have open. You need
1231 # to make sure that no handles to the write end of the
1232 # output pipe are maintained in this process or else the
1233 # pipe will not close when the child process exits and the
1234 # ReadFile will hang.
1235 if p2cread != -1:
1236 p2cread.Close()
1237 if c2pwrite != -1:
1238 c2pwrite.Close()
1239 if errwrite != -1:
1240 errwrite.Close()
1241 if hasattr(self, '_devnull'):
1242 os.close(self._devnull)
1243
1244 # Retain the process handle, but close the thread handle
1245 self._child_created = True
1246 self._handle = Handle(hp)
1247 self.pid = pid
1248 _winapi.CloseHandle(ht)
1249
1250 def _internal_poll(self, _deadstate=None,
1251 _WaitForSingleObject=_winapi.WaitForSingleObject,
1252 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
1253 _GetExitCodeProcess=_winapi.GetExitCodeProcess):
1254 """Check if child process has terminated. Returns returncode
1255 attribute.
1256
1257 This method is called by __del__, so it can only refer to objects
1258 in its local scope.
1259
1260 """
1261 if self.returncode is None:
1262 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0:
1263 self.returncode = _GetExitCodeProcess(self._handle)
1264 return self.returncode
1265
1266
1267 def wait(self, timeout=None, endtime=None):
1268 """Wait for child process to terminate. Returns returncode
1269 attribute."""
1270 if endtime is not None:
1271 timeout = self._remaining_time(endtime)
1272 if timeout is None:
1273 timeout_millis = _winapi.INFINITE
1274 else:
1275 timeout_millis = int(timeout * 1000)
1276 if self.returncode is None:
1277 result = _winapi.WaitForSingleObject(self._handle,
1278 timeout_millis)
1279 if result == _winapi.WAIT_TIMEOUT:
1280 raise TimeoutExpired(self.args, timeout)
1281 self.returncode = _winapi.GetExitCodeProcess(self._handle)
1282 return self.returncode
1283
1284
1285 def _readerthread(self, fh, buffer):
1286 buffer.append(fh.read())
1287 fh.close()
1288
1289
1290 def _communicate(self, input, endtime, orig_timeout):
1291 # Start reader threads feeding into a list hanging off of this
1292 # object, unless they've already been started.
1293 if self.stdout and not hasattr(self, "_stdout_buff"):
1294 self._stdout_buff = []
1295 self.stdout_thread = \
1296 threading.Thread(target=self._readerthread,
1297 args=(self.stdout, self._stdout_buff))
1298 self.stdout_thread.daemon = True
1299 self.stdout_thread.start()
1300 if self.stderr and not hasattr(self, "_stderr_buff"):
1301 self._stderr_buff = []
1302 self.stderr_thread = \
1303 threading.Thread(target=self._readerthread,
1304 args=(self.stderr, self._stderr_buff))
1305 self.stderr_thread.daemon = True
1306 self.stderr_thread.start()
1307
1308 if self.stdin:
1309 self._stdin_write(input)
1310
1311 # Wait for the reader threads, or time out. If we time out, the
1312 # threads remain reading and the fds left open in case the user
1313 # calls communicate again.
1314 if self.stdout is not None:
1315 self.stdout_thread.join(self._remaining_time(endtime))
1316 if self.stdout_thread.is_alive():
1317 raise TimeoutExpired(self.args, orig_timeout)
1318 if self.stderr is not None:
1319 self.stderr_thread.join(self._remaining_time(endtime))
1320 if self.stderr_thread.is_alive():
1321 raise TimeoutExpired(self.args, orig_timeout)
1322
1323 # Collect the output from and close both pipes, now that we know
1324 # both have been read successfully.
1325 stdout = None
1326 stderr = None
1327 if self.stdout: 1000 if self.stdout:
1328 stdout = self._stdout_buff
1329 self.stdout.close() 1001 self.stdout.close()
1330 if self.stderr: 1002 if self.stderr:
1331 stderr = self._stderr_buff
1332 self.stderr.close() 1003 self.stderr.close()
1333 1004 try: # Flushing a BufferedWriter may raise an error
1334 # All data exchanged. Translate lists into strings. 1005 if self.stdin:
1335 if stdout is not None: 1006 self.stdin.close()
1336 stdout = stdout[0] 1007 finally:
1337 if stderr is not None: 1008 # Wait for the process to terminate, to avoid zombies.
1338 stderr = stderr[0] 1009 self.wait()
1010
1011 def __del__(self, _maxsize=sys.maxsize):
1012 if not self._child_created:
1013 # We didn't get to successfully create a child process.
1014 return
1015 # In case the child hasn't been waited on, check if it's done.
1016 self._internal_poll(_deadstate=_maxsize)
1017 if self.returncode is None and _active is not None:
1018 # Child is still running, keep us alive until we can wait on it.
1019 _active.append(self)
1020
1021 def _get_devnull(self):
1022 if not hasattr(self, '_devnull'):
1023 self._devnull = os.open(os.devnull, os.O_RDWR)
1024 return self._devnull
1025
1026 def _stdin_write(self, input):
1027 if input:
1028 try:
1029 self.stdin.write(input)
1030 except BrokenPipeError:
1031 # communicate() must ignore broken pipe error
1032 pass
1033 except OSError as e:
1034 if e.errno == errno.EINVAL and self.poll() is not None:
1035 # Issue #19612: On Windows, stdin.write() fails with EIN VAL
1036 # if the process already exited before the write
1037 pass
1038 else:
1039 raise
1040 self.stdin.close()
1041
1042 def communicate(self, input=None, timeout=None):
1043 """Interact with process: Send data to stdin. Read data from
1044 stdout and stderr, until end-of-file is reached. Wait for
1045 process to terminate.
1046
1047 The optional "input" argument should be data to be sent to the
1048 child process (if self.universal_newlines is True, this should
1049 be a string; if it is False, "input" should be bytes), or
1050 None, if no data should be sent to the child.
1051
1052 communicate() returns a tuple (stdout, stderr). These will be
1053 bytes or, if self.universal_newlines was True, a string.
1054 """
1055
1056 if self._communication_started and input:
1057 raise ValueError("Cannot send input after starting communication ")
1058
1059 # Optimization: If we are not worried about timeouts, we haven't
1060 # started communicating, and we have one or zero pipes, using select ()
1061 # or threads is unnecessary.
1062 if (timeout is None and not self._communication_started and
1063 [self.stdin, self.stdout, self.stderr].count(None) >= 2):
1064 stdout = None
1065 stderr = None
1066 if self.stdin:
1067 self._stdin_write(input)
1068 elif self.stdout:
1069 stdout = self.stdout.read()
1070 self.stdout.close()
1071 elif self.stderr:
1072 stderr = self.stderr.read()
1073 self.stderr.close()
1074 self.wait()
1075 else:
1076 if timeout is not None:
1077 endtime = _time() + timeout
1078 else:
1079 endtime = None
1080
1081 try:
1082 stdout, stderr = self._communicate(input, endtime, timeout)
1083 finally:
1084 self._communication_started = True
1085
1086 sts = self.wait(timeout=self._remaining_time(endtime))
1339 1087
1340 return (stdout, stderr) 1088 return (stdout, stderr)
1341 1089
1342 def send_signal(self, sig): 1090
1343 """Send a signal to the process.""" 1091 def poll(self):
1344 # Don't signal a process that we know has already died. 1092 return self._internal_poll()
1345 if self.returncode is not None: 1093
1094
1095 def _remaining_time(self, endtime):
1096 """Convenience for _communicate when computing timeouts."""
1097 if endtime is None:
1098 return None
1099 else:
1100 return endtime - _time()
1101
1102
1103 def _check_timeout(self, endtime, orig_timeout):
1104 """Convenience for checking if a timeout has expired."""
1105 if endtime is None:
1346 return 1106 return
1347 if sig == signal.SIGTERM: 1107 if _time() > endtime:
1348 self.terminate() 1108 raise TimeoutExpired(self.args, orig_timeout)
1349 elif sig == signal.CTRL_C_EVENT: 1109
1350 os.kill(self.pid, signal.CTRL_C_EVENT) 1110
1351 elif sig == signal.CTRL_BREAK_EVENT: 1111 if _mswindows:
1352 os.kill(self.pid, signal.CTRL_BREAK_EVENT) 1112 #
1353 else: 1113 # Windows methods
1354 raise ValueError("Unsupported signal: {}".format(sig)) 1114 #
1355 1115 def _get_handles(self, stdin, stdout, stderr):
1356 def terminate(self): 1116 """Construct and return tuple with IO objects:
1357 """Terminates the process.""" 1117 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1358 # Don't terminate a process that we know has already died. 1118 """
1359 if self.returncode is not None: 1119 if stdin is None and stdout is None and stderr is None:
1360 return 1120 return (-1, -1, -1, -1, -1, -1)
1361 try: 1121
1362 _winapi.TerminateProcess(self._handle, 1) 1122 p2cread, p2cwrite = -1, -1
1363 except PermissionError: 1123 c2pread, c2pwrite = -1, -1
1364 # ERROR_ACCESS_DENIED (winerror 5) is received when the 1124 errread, errwrite = -1, -1
1365 # process already died. 1125
1366 rc = _winapi.GetExitCodeProcess(self._handle) 1126 if stdin is None:
1367 if rc == _winapi.STILL_ACTIVE: 1127 p2cread = _winapi.GetStdHandle(_winapi.STD_INPUT_HANDLE)
1368 raise 1128 if p2cread is None:
1369 self.returncode = rc 1129 p2cread, _ = _winapi.CreatePipe(None, 0)
1370 1130 p2cread = Handle(p2cread)
1371 kill = terminate 1131 _winapi.CloseHandle(_)
1372 1132 elif stdin == PIPE:
1373 else: 1133 p2cread, p2cwrite = _winapi.CreatePipe(None, 0)
1374 # 1134 p2cread, p2cwrite = Handle(p2cread), Handle(p2cwrite)
1375 # POSIX methods 1135 elif stdin == DEVNULL:
1376 # 1136 p2cread = msvcrt.get_osfhandle(self._get_devnull())
1377 def _get_handles(self, stdin, stdout, stderr): 1137 elif isinstance(stdin, int):
1378 """Construct and return tuple with IO objects: 1138 p2cread = msvcrt.get_osfhandle(stdin)
1379 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite 1139 else:
1380 """ 1140 # Assuming file-like object
1381 p2cread, p2cwrite = -1, -1 1141 p2cread = msvcrt.get_osfhandle(stdin.fileno())
1382 c2pread, c2pwrite = -1, -1 1142 p2cread = self._make_inheritable(p2cread)
1383 errread, errwrite = -1, -1 1143
1384 1144 if stdout is None:
1385 if stdin is None: 1145 c2pwrite = _winapi.GetStdHandle(_winapi.STD_OUTPUT_HANDLE)
1386 pass 1146 if c2pwrite is None:
1387 elif stdin == PIPE: 1147 _, c2pwrite = _winapi.CreatePipe(None, 0)
1388 p2cread, p2cwrite = os.pipe() 1148 c2pwrite = Handle(c2pwrite)
1389 elif stdin == DEVNULL: 1149 _winapi.CloseHandle(_)
1390 p2cread = self._get_devnull() 1150 elif stdout == PIPE:
1391 elif isinstance(stdin, int): 1151 c2pread, c2pwrite = _winapi.CreatePipe(None, 0)
1392 p2cread = stdin 1152 c2pread, c2pwrite = Handle(c2pread), Handle(c2pwrite)
1393 else: 1153 elif stdout == DEVNULL:
1394 # Assuming file-like object 1154 c2pwrite = msvcrt.get_osfhandle(self._get_devnull())
1395 p2cread = stdin.fileno() 1155 elif isinstance(stdout, int):
1396 1156 c2pwrite = msvcrt.get_osfhandle(stdout)
1397 if stdout is None: 1157 else:
1398 pass 1158 # Assuming file-like object
1399 elif stdout == PIPE: 1159 c2pwrite = msvcrt.get_osfhandle(stdout.fileno())
1400 c2pread, c2pwrite = os.pipe() 1160 c2pwrite = self._make_inheritable(c2pwrite)
1401 elif stdout == DEVNULL: 1161
1402 c2pwrite = self._get_devnull() 1162 if stderr is None:
1403 elif isinstance(stdout, int): 1163 errwrite = _winapi.GetStdHandle(_winapi.STD_ERROR_HANDLE)
1404 c2pwrite = stdout 1164 if errwrite is None:
1405 else: 1165 _, errwrite = _winapi.CreatePipe(None, 0)
1406 # Assuming file-like object 1166 errwrite = Handle(errwrite)
1407 c2pwrite = stdout.fileno() 1167 _winapi.CloseHandle(_)
1408 1168 elif stderr == PIPE:
1409 if stderr is None: 1169 errread, errwrite = _winapi.CreatePipe(None, 0)
1410 pass 1170 errread, errwrite = Handle(errread), Handle(errwrite)
1411 elif stderr == PIPE: 1171 elif stderr == STDOUT:
1412 errread, errwrite = os.pipe() 1172 errwrite = c2pwrite
1413 elif stderr == STDOUT: 1173 elif stderr == DEVNULL:
1414 errwrite = c2pwrite 1174 errwrite = msvcrt.get_osfhandle(self._get_devnull())
1415 elif stderr == DEVNULL: 1175 elif isinstance(stderr, int):
1416 errwrite = self._get_devnull() 1176 errwrite = msvcrt.get_osfhandle(stderr)
1417 elif isinstance(stderr, int): 1177 else:
1418 errwrite = stderr 1178 # Assuming file-like object
1419 else: 1179 errwrite = msvcrt.get_osfhandle(stderr.fileno())
1420 # Assuming file-like object 1180 errwrite = self._make_inheritable(errwrite)
1421 errwrite = stderr.fileno() 1181
1422 1182 return (p2cread, p2cwrite,
1423 return (p2cread, p2cwrite, 1183 c2pread, c2pwrite,
1424 c2pread, c2pwrite, 1184 errread, errwrite)
1425 errread, errwrite) 1185
1426 1186
1427 1187 def _make_inheritable(self, handle):
1428 def _execute_child(self, args, executable, preexec_fn, close_fds, 1188 """Return a duplicate of handle, which is inheritable"""
1429 pass_fds, cwd, env, 1189 h = _winapi.DuplicateHandle(
1430 startupinfo, creationflags, shell, 1190 _winapi.GetCurrentProcess(), handle,
1431 p2cread, p2cwrite, 1191 _winapi.GetCurrentProcess(), 0, 1,
1432 c2pread, c2pwrite, 1192 _winapi.DUPLICATE_SAME_ACCESS)
1433 errread, errwrite, 1193 return Handle(h)
1434 restore_signals, start_new_session): 1194
1435 """Execute program (POSIX version)""" 1195
1436 1196 def _execute_child(self, args, executable, preexec_fn, close_fds,
1437 if isinstance(args, (str, bytes)): 1197 pass_fds, cwd, env,
1438 args = [args] 1198 startupinfo, creationflags, shell,
1439 else: 1199 p2cread, p2cwrite,
1440 args = list(args) 1200 c2pread, c2pwrite,
1441 1201 errread, errwrite,
1442 if shell: 1202 unused_restore_signals, unused_start_new_session) :
1443 args = ["/bin/sh", "-c"] + args 1203 """Execute program (MS Windows version)"""
1444 if executable: 1204
1445 args[0] = executable 1205 assert not pass_fds, "pass_fds not supported on Windows."
1446 1206
1447 if executable is None: 1207 if not isinstance(args, str):
1448 executable = args[0] 1208 args = list2cmdline(args)
1449 orig_executable = executable 1209
1450 1210 # Process startup details
1451 # For transferring possible exec failure from child to parent. 1211 if startupinfo is None:
1452 # Data format: "exception name:hex errno:description" 1212 startupinfo = STARTUPINFO()
1453 # Pickle is not used; it is complex and involves memory allocation. 1213 if -1 not in (p2cread, c2pwrite, errwrite):
1454 errpipe_read, errpipe_write = os.pipe() 1214 startupinfo.dwFlags |= _winapi.STARTF_USESTDHANDLES
1455 # errpipe_write must not be in the standard io 0, 1, or 2 fd range. 1215 startupinfo.hStdInput = p2cread
1456 low_fds_to_close = [] 1216 startupinfo.hStdOutput = c2pwrite
1457 while errpipe_write < 3: 1217 startupinfo.hStdError = errwrite
1458 low_fds_to_close.append(errpipe_write) 1218
1459 errpipe_write = os.dup(errpipe_write) 1219 if shell:
1460 for low_fd in low_fds_to_close: 1220 startupinfo.dwFlags |= _winapi.STARTF_USESHOWWINDOW
1461 os.close(low_fd) 1221 startupinfo.wShowWindow = _winapi.SW_HIDE
1462 try: 1222 comspec = os.environ.get("COMSPEC", "cmd.exe")
1223 args = '{} /c "{}"'.format (comspec, args)
1224
1225 # Start the process
1463 try: 1226 try:
1464 # We must avoid complex work that could involve 1227 hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
1465 # malloc or free in the child process to avoid 1228 # no special security
1466 # potential deadlocks, thus we do all this here. 1229 None, None,
1467 # and pass it to fork_exec() 1230 int(not close_fds),
1468 1231 creationflags,
1469 if env is not None: 1232 env,
1470 env_list = [os.fsencode(k) + b'=' + os.fsencode(v) 1233 cwd,
1471 for k, v in env.items()] 1234 startupinfo)
1472 else: 1235 finally:
1473 env_list = None # Use execv instead of execve. 1236 # Child is launched. Close the parent's copy of those pipe
1474 executable = os.fsencode(executable) 1237 # handles that only the child should have open. You need
1475 if os.path.dirname(executable): 1238 # to make sure that no handles to the write end of the
1476 executable_list = (executable,) 1239 # output pipe are maintained in this process or else the
1477 else: 1240 # pipe will not close when the child process exits and the
1478 # This matches the behavior of os._execvpe(). 1241 # ReadFile will hang.
1479 executable_list = tuple( 1242 if p2cread != -1:
1480 os.path.join(os.fsencode(dir), executable) 1243 p2cread.Close()
1481 for dir in os.get_exec_path(env)) 1244 if c2pwrite != -1:
1482 fds_to_keep = set(pass_fds) 1245 c2pwrite.Close()
1483 fds_to_keep.add(errpipe_write) 1246 if errwrite != -1:
1484 self.pid = _posixsubprocess.fork_exec( 1247 errwrite.Close()
1485 args, executable_list, 1248 if hasattr(self, '_devnull'):
1486 close_fds, sorted(fds_to_keep), cwd, env_list, 1249 os.close(self._devnull)
1487 p2cread, p2cwrite, c2pread, c2pwrite, 1250
1488 errread, errwrite, 1251 # Retain the process handle, but close the thread handle
1489 errpipe_read, errpipe_write, 1252 self._child_created = True
1490 restore_signals, start_new_session, preexec_fn) 1253 self._handle = Handle(hp)
1491 self._child_created = True 1254 self.pid = pid
1255 _winapi.CloseHandle(ht)
1256
1257 def _internal_poll(self, _deadstate=None,
1258 _WaitForSingleObject=_winapi.WaitForSingleObject,
1259 _WAIT_OBJECT_0=_winapi.WAIT_OBJECT_0,
1260 _GetExitCodeProcess=_winapi.GetExitCodeProcess):
1261 """Check if child process has terminated. Returns returncode
1262 attribute.
1263
1264 This method is called by __del__, so it can only refer to object s
1265 in its local scope.
1266
1267 """
1268 if self.returncode is None:
1269 if _WaitForSingleObject(self._handle, 0) == _WAIT_OBJECT_0:
1270 self.returncode = _GetExitCodeProcess(self._handle)
1271 return self.returncode
1272
1273
1274 def wait(self, timeout=None, endtime=None):
1275 """Wait for child process to terminate. Returns returncode
1276 attribute."""
1277 if endtime is not None:
1278 timeout = self._remaining_time(endtime)
1279 if timeout is None:
1280 timeout_millis = _winapi.INFINITE
1281 else:
1282 timeout_millis = int(timeout * 1000)
1283 if self.returncode is None:
1284 result = _winapi.WaitForSingleObject(self._handle,
1285 timeout_millis)
1286 if result == _winapi.WAIT_TIMEOUT:
1287 raise TimeoutExpired(self.args, timeout)
1288 self.returncode = _winapi.GetExitCodeProcess(self._handle)
1289 return self.returncode
1290
1291
1292 def _readerthread(self, fh, buffer):
1293 buffer.append(fh.read())
1294 fh.close()
1295
1296
1297 def _communicate(self, input, endtime, orig_timeout):
1298 # Start reader threads feeding into a list hanging off of this
1299 # object, unless they've already been started.
1300 if self.stdout and not hasattr(self, "_stdout_buff"):
1301 self._stdout_buff = []
1302 self.stdout_thread = \
1303 threading.Thread(target=self._readerthread,
1304 args=(self.stdout, self._stdout_buf f))
1305 self.stdout_thread.daemon = True
1306 self.stdout_thread.start()
1307 if self.stderr and not hasattr(self, "_stderr_buff"):
1308 self._stderr_buff = []
1309 self.stderr_thread = \
1310 threading.Thread(target=self._readerthread,
1311 args=(self.stderr, self._stderr_buf f))
1312 self.stderr_thread.daemon = True
1313 self.stderr_thread.start()
1314
1315 if self.stdin:
1316 self._stdin_write(input)
1317
1318 # Wait for the reader threads, or time out. If we time out, the
1319 # threads remain reading and the fds left open in case the user
1320 # calls communicate again.
1321 if self.stdout is not None:
1322 self.stdout_thread.join(self._remaining_time(endtime))
1323 if self.stdout_thread.is_alive():
1324 raise TimeoutExpired(self.args, orig_timeout)
1325 if self.stderr is not None:
1326 self.stderr_thread.join(self._remaining_time(endtime))
1327 if self.stderr_thread.is_alive():
1328 raise TimeoutExpired(self.args, orig_timeout)
1329
1330 # Collect the output from and close both pipes, now that we know
1331 # both have been read successfully.
1332 stdout = None
1333 stderr = None
1334 if self.stdout:
1335 stdout = self._stdout_buff
1336 self.stdout.close()
1337 if self.stderr:
1338 stderr = self._stderr_buff
1339 self.stderr.close()
1340
1341 # All data exchanged. Translate lists into strings.
1342 if stdout is not None:
1343 stdout = stdout[0]
1344 if stderr is not None:
1345 stderr = stderr[0]
1346
1347 return (stdout, stderr)
1348
1349 def send_signal(self, sig):
1350 """Send a signal to the process."""
1351 # Don't signal a process that we know has already died.
1352 if self.returncode is not None:
1353 return
1354 if sig == signal.SIGTERM:
1355 self.terminate()
1356 elif sig == signal.CTRL_C_EVENT:
1357 os.kill(self.pid, signal.CTRL_C_EVENT)
1358 elif sig == signal.CTRL_BREAK_EVENT:
1359 os.kill(self.pid, signal.CTRL_BREAK_EVENT)
1360 else:
1361 raise ValueError("Unsupported signal: {}".format(sig))
1362
1363 def terminate(self):
1364 """Terminates the process."""
1365 # Don't terminate a process that we know has already died.
1366 if self.returncode is not None:
1367 return
1368 try:
1369 _winapi.TerminateProcess(self._handle, 1)
1370 except PermissionError:
1371 # ERROR_ACCESS_DENIED (winerror 5) is received when the
1372 # process already died.
1373 rc = _winapi.GetExitCodeProcess(self._handle)
1374 if rc == _winapi.STILL_ACTIVE:
1375 raise
1376 self.returncode = rc
1377
1378 kill = terminate
1379
1380 else:
1381 #
1382 # POSIX methods
1383 #
1384 def _get_handles(self, stdin, stdout, stderr):
1385 """Construct and return tuple with IO objects:
1386 p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
1387 """
1388 p2cread, p2cwrite = -1, -1
1389 c2pread, c2pwrite = -1, -1
1390 errread, errwrite = -1, -1
1391
1392 if stdin is None:
1393 pass
1394 elif stdin == PIPE:
1395 p2cread, p2cwrite = os.pipe()
1396 elif stdin == DEVNULL:
1397 p2cread = self._get_devnull()
1398 elif isinstance(stdin, int):
1399 p2cread = stdin
1400 else:
1401 # Assuming file-like object
1402 p2cread = stdin.fileno()
1403
1404 if stdout is None:
1405 pass
1406 elif stdout == PIPE:
1407 c2pread, c2pwrite = os.pipe()
1408 elif stdout == DEVNULL:
1409 c2pwrite = self._get_devnull()
1410 elif isinstance(stdout, int):
1411 c2pwrite = stdout
1412 else:
1413 # Assuming file-like object
1414 c2pwrite = stdout.fileno()
1415
1416 if stderr is None:
1417 pass
1418 elif stderr == PIPE:
1419 errread, errwrite = os.pipe()
1420 elif stderr == STDOUT:
1421 errwrite = c2pwrite
1422 elif stderr == DEVNULL:
1423 errwrite = self._get_devnull()
1424 elif isinstance(stderr, int):
1425 errwrite = stderr
1426 else:
1427 # Assuming file-like object
1428 errwrite = stderr.fileno()
1429
1430 return (p2cread, p2cwrite,
1431 c2pread, c2pwrite,
1432 errread, errwrite)
1433
1434
1435 def _execute_child(self, args, executable, preexec_fn, close_fds,
1436 pass_fds, cwd, env,
1437 startupinfo, creationflags, shell,
1438 p2cread, p2cwrite,
1439 c2pread, c2pwrite,
1440 errread, errwrite,
1441 restore_signals, start_new_session):
1442 """Execute program (POSIX version)"""
1443
1444 if isinstance(args, (str, bytes)):
1445 args = [args]
1446 else:
1447 args = list(args)
1448
1449 if shell:
1450 args = ["/bin/sh", "-c"] + args
1451 if executable:
1452 args[0] = executable
1453
1454 if executable is None:
1455 executable = args[0]
1456 orig_executable = executable
1457
1458 # For transferring possible exec failure from child to parent.
1459 # Data format: "exception name:hex errno:description"
1460 # Pickle is not used; it is complex and involves memory allocati on.
1461 errpipe_read, errpipe_write = os.pipe()
1462 # errpipe_write must not be in the standard io 0, 1, or 2 fd ran ge.
1463 low_fds_to_close = []
1464 while errpipe_write < 3:
1465 low_fds_to_close.append(errpipe_write)
1466 errpipe_write = os.dup(errpipe_write)
1467 for low_fd in low_fds_to_close:
1468 os.close(low_fd)
1469 try:
1470 try:
1471 # We must avoid complex work that could involve
1472 # malloc or free in the child process to avoid
1473 # potential deadlocks, thus we do all this here.
1474 # and pass it to fork_exec()
1475
1476 if env is not None:
1477 env_list = [os.fsencode(k) + b'=' + os.fsencode(v)
1478 for k, v in env.items()]
1479 else:
1480 env_list = None # Use execv instead of execve.
1481 executable = os.fsencode(executable)
1482 if os.path.dirname(executable):
1483 executable_list = (executable,)
1484 else:
1485 # This matches the behavior of os._execvpe().
1486 executable_list = tuple(
1487 os.path.join(os.fsencode(dir), executable)
1488 for dir in os.get_exec_path(env))
1489 fds_to_keep = set(pass_fds)
1490 fds_to_keep.add(errpipe_write)
1491 self.pid = _posixsubprocess.fork_exec(
1492 args, executable_list,
1493 close_fds, sorted(fds_to_keep), cwd, env_list,
1494 p2cread, p2cwrite, c2pread, c2pwrite,
1495 errread, errwrite,
1496 errpipe_read, errpipe_write,
1497 restore_signals, start_new_session, preexec_fn)
1498 self._child_created = True
1499 finally:
1500 # be sure the FD is closed no matter what
1501 os.close(errpipe_write)
1502
1503 # self._devnull is not always defined.
1504 devnull_fd = getattr(self, '_devnull', None)
1505 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_f d:
1506 os.close(p2cread)
1507 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_ fd:
1508 os.close(c2pwrite)
1509 if errwrite != -1 and errread != -1 and errwrite != devnull_ fd:
1510 os.close(errwrite)
1511 if devnull_fd is not None:
1512 os.close(devnull_fd)
1513 # Prevent a double close of these fds from __init__ on error .
1514 self._closed_child_pipe_fds = True
1515
1516 # Wait for exec to fail or succeed; possibly raising an
1517 # exception (limited in size)
1518 errpipe_data = bytearray()
1519 while True:
1520 part = os.read(errpipe_read, 50000)
1521 errpipe_data += part
1522 if not part or len(errpipe_data) > 50000:
1523 break
1492 finally: 1524 finally:
1493 # be sure the FD is closed no matter what 1525 # be sure the FD is closed no matter what
1494 os.close(errpipe_write) 1526 os.close(errpipe_read)
1495 1527
1496 # self._devnull is not always defined. 1528 if errpipe_data:
1497 devnull_fd = getattr(self, '_devnull', None) 1529 try:
1498 if p2cread != -1 and p2cwrite != -1 and p2cread != devnull_fd: 1530 os.waitpid(self.pid, 0)
1499 os.close(p2cread) 1531 except ChildProcessError:
1500 if c2pwrite != -1 and c2pread != -1 and c2pwrite != devnull_fd: 1532 pass
1501 os.close(c2pwrite) 1533 try:
1502 if errwrite != -1 and errread != -1 and errwrite != devnull_fd: 1534 exception_name, hex_errno, err_msg = (
1503 os.close(errwrite) 1535 errpipe_data.split(b':', 2))
1504 if devnull_fd is not None: 1536 except ValueError:
1505 os.close(devnull_fd) 1537 exception_name = b'SubprocessError'
1506 # Prevent a double close of these fds from __init__ on error. 1538 hex_errno = b'0'
1507 self._closed_child_pipe_fds = True 1539 err_msg = (b'Bad exception data from child: ' +
1508 1540 repr(errpipe_data))
1509 # Wait for exec to fail or succeed; possibly raising an 1541 child_exception_type = getattr(
1510 # exception (limited in size) 1542 builtins, exception_name.decode('ascii'),
1511 errpipe_data = bytearray() 1543 SubprocessError)
1512 while True: 1544 err_msg = err_msg.decode(errors="surrogatepass")
1513 part = os.read(errpipe_read, 50000) 1545 if issubclass(child_exception_type, OSError) and hex_errno:
1514 errpipe_data += part 1546 errno_num = int(hex_errno, 16)
1515 if not part or len(errpipe_data) > 50000: 1547 child_exec_never_called = (err_msg == "noexec")
1516 break 1548 if child_exec_never_called:
1517 finally: 1549 err_msg = ""
1518 # be sure the FD is closed no matter what 1550 if errno_num != 0:
1519 os.close(errpipe_read) 1551 err_msg = os.strerror(errno_num)
1520 1552 if errno_num == errno.ENOENT:
1521 if errpipe_data: 1553 if child_exec_never_called:
1554 # The error must be from chdir(cwd).
1555 err_msg += ': ' + repr(cwd)
1556 else:
1557 err_msg += ': ' + repr(orig_executable)
1558 raise child_exception_type(errno_num, err_msg)
1559 raise child_exception_type(err_msg)
1560
1561
1562 def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED,
1563 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED,
1564 _WEXITSTATUS=os.WEXITSTATUS):
1565 """All callers to this function MUST hold self._waitpid_lock."""
1566 # This method is called (indirectly) by __del__, so it cannot
1567 # refer to anything outside of its local scope.
1568 if _WIFSIGNALED(sts):
1569 self.returncode = -_WTERMSIG(sts)
1570 elif _WIFEXITED(sts):
1571 self.returncode = _WEXITSTATUS(sts)
1572 else:
1573 # Should never happen
1574 raise SubprocessError("Unknown child exit status!")
1575
1576
1577 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid,
1578 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
1579 """Check if child process has terminated. Returns returncode
1580 attribute.
1581
1582 This method is called by __del__, so it cannot reference anythin g
1583 outside of the local scope (nor can any methods it calls).
1584
1585 """
1586 if self.returncode is None:
1587 if not self._waitpid_lock.acquire(False):
1588 # Something else is busy calling waitpid. Don't allow t wo
1589 # at once. We know nothing yet.
1590 return None
1591 try:
1592 if self.returncode is not None:
1593 return self.returncode # Another thread waited.
1594 pid, sts = _waitpid(self.pid, _WNOHANG)
1595 if pid == self.pid:
1596 self._handle_exitstatus(sts)
1597 except OSError as e:
1598 if _deadstate is not None:
1599 self.returncode = _deadstate
1600 elif e.errno == _ECHILD:
1601 # This happens if SIGCLD is set to be ignored or
1602 # waiting for child processes has otherwise been
1603 # disabled for our process. This child is dead, we
1604 # can't get the status.
1605 # http://bugs.python.org/issue15756
1606 self.returncode = 0
1607 finally:
1608 self._waitpid_lock.release()
1609 return self.returncode
1610
1611
1612 def _try_wait(self, wait_flags):
1613 """All callers to this function MUST hold self._waitpid_lock."""
1522 try: 1614 try:
1523 os.waitpid(self.pid, 0) 1615 (pid, sts) = os.waitpid(self.pid, wait_flags)
1524 except ChildProcessError: 1616 except ChildProcessError:
1525 pass 1617 # This happens if SIGCLD is set to be ignored or waiting
1526 try: 1618 # for child processes has otherwise been disabled for our
1527 exception_name, hex_errno, err_msg = ( 1619 # process. This child is dead, we can't get the status.
1528 errpipe_data.split(b':', 2)) 1620 pid = self.pid
1529 except ValueError: 1621 sts = 0
1530 exception_name = b'SubprocessError' 1622 return (pid, sts)
1531 hex_errno = b'0' 1623
1532 err_msg = (b'Bad exception data from child: ' + 1624
1533 repr(errpipe_data)) 1625 def wait(self, timeout=None, endtime=None):
1534 child_exception_type = getattr( 1626 """Wait for child process to terminate. Returns returncode
1535 builtins, exception_name.decode('ascii'), 1627 attribute."""
1536 SubprocessError) 1628 if self.returncode is not None:
1537 err_msg = err_msg.decode(errors="surrogatepass") 1629 return self.returncode
1538 if issubclass(child_exception_type, OSError) and hex_errno: 1630
1539 errno_num = int(hex_errno, 16) 1631 # endtime is preferred to timeout. timeout is only used for
1540 child_exec_never_called = (err_msg == "noexec") 1632 # printing.
1541 if child_exec_never_called: 1633 if endtime is not None or timeout is not None:
1542 err_msg = "" 1634 if endtime is None:
1543 if errno_num != 0: 1635 endtime = _time() + timeout
1544 err_msg = os.strerror(errno_num) 1636 elif timeout is None:
1545 if errno_num == errno.ENOENT: 1637 timeout = self._remaining_time(endtime)
1546 if child_exec_never_called: 1638
1547 # The error must be from chdir(cwd). 1639 if endtime is not None:
1548 err_msg += ': ' + repr(cwd) 1640 # Enter a busy loop if we have a timeout. This busy loop wa s
1549 else: 1641 # cribbed from Lib/threading.py in Thread.wait() at r71065.
1550 err_msg += ': ' + repr(orig_executable) 1642 delay = 0.0005 # 500 us -> initial delay of 1 ms
1551 raise child_exception_type(errno_num, err_msg) 1643 while True:
1552 raise child_exception_type(err_msg) 1644 if self._waitpid_lock.acquire(False):
1553 1645 try:
1554 1646 if self.returncode is not None:
1555 def _handle_exitstatus(self, sts, _WIFSIGNALED=os.WIFSIGNALED, 1647 break # Another thread waited.
1556 _WTERMSIG=os.WTERMSIG, _WIFEXITED=os.WIFEXITED, 1648 (pid, sts) = self._try_wait(os.WNOHANG)
1557 _WEXITSTATUS=os.WEXITSTATUS): 1649 assert pid == self.pid or pid == 0
1558 """All callers to this function MUST hold self._waitpid_lock.""" 1650 if pid == self.pid:
1559 # This method is called (indirectly) by __del__, so it cannot 1651 self._handle_exitstatus(sts)
1560 # refer to anything outside of its local scope. 1652 break
1561 if _WIFSIGNALED(sts): 1653 finally:
1562 self.returncode = -_WTERMSIG(sts) 1654 self._waitpid_lock.release()
1563 elif _WIFEXITED(sts): 1655 remaining = self._remaining_time(endtime)
1564 self.returncode = _WEXITSTATUS(sts) 1656 if remaining <= 0:
1565 else: 1657 raise TimeoutExpired(self.args, timeout)
1566 # Should never happen 1658 delay = min(delay * 2, remaining, .05)
1567 raise SubprocessError("Unknown child exit status!") 1659 time.sleep(delay)
1568 1660 else:
1569 1661 while self.returncode is None:
1570 def _internal_poll(self, _deadstate=None, _waitpid=os.waitpid, 1662 with self._waitpid_lock:
1571 _WNOHANG=os.WNOHANG, _ECHILD=errno.ECHILD):
1572 """Check if child process has terminated. Returns returncode
1573 attribute.
1574
1575 This method is called by __del__, so it cannot reference anything
1576 outside of the local scope (nor can any methods it calls).
1577
1578 """
1579 if self.returncode is None:
1580 if not self._waitpid_lock.acquire(False):
1581 # Something else is busy calling waitpid. Don't allow two
1582 # at once. We know nothing yet.
1583 return None
1584 try:
1585 if self.returncode is not None:
1586 return self.returncode # Another thread waited.
1587 pid, sts = _waitpid(self.pid, _WNOHANG)
1588 if pid == self.pid:
1589 self._handle_exitstatus(sts)
1590 except OSError as e:
1591 if _deadstate is not None:
1592 self.returncode = _deadstate
1593 elif e.errno == _ECHILD:
1594 # This happens if SIGCLD is set to be ignored or
1595 # waiting for child processes has otherwise been
1596 # disabled for our process. This child is dead, we
1597 # can't get the status.
1598 # http://bugs.python.org/issue15756
1599 self.returncode = 0
1600 finally:
1601 self._waitpid_lock.release()
1602 return self.returncode
1603
1604
1605 def _try_wait(self, wait_flags):
1606 """All callers to this function MUST hold self._waitpid_lock."""
1607 try:
1608 (pid, sts) = os.waitpid(self.pid, wait_flags)
1609 except ChildProcessError:
1610 # This happens if SIGCLD is set to be ignored or waiting
1611 # for child processes has otherwise been disabled for our
1612 # process. This child is dead, we can't get the status.
1613 pid = self.pid
1614 sts = 0
1615 return (pid, sts)
1616
1617
1618 def wait(self, timeout=None, endtime=None):
1619 """Wait for child process to terminate. Returns returncode
1620 attribute."""
1621 if self.returncode is not None:
1622 return self.returncode
1623
1624 # endtime is preferred to timeout. timeout is only used for
1625 # printing.
1626 if endtime is not None or timeout is not None:
1627 if endtime is None:
1628 endtime = _time() + timeout
1629 elif timeout is None:
1630 timeout = self._remaining_time(endtime)
1631
1632 if endtime is not None:
1633 # Enter a busy loop if we have a timeout. This busy loop was
1634 # cribbed from Lib/threading.py in Thread.wait() at r71065.
1635 delay = 0.0005 # 500 us -> initial delay of 1 ms
1636 while True:
1637 if self._waitpid_lock.acquire(False):
1638 try:
1639 if self.returncode is not None: 1663 if self.returncode is not None:
1640 break # Another thread waited. 1664 break # Another thread waited.
1641 (pid, sts) = self._try_wait(os.WNOHANG) 1665 (pid, sts) = self._try_wait(0)
1642 assert pid == self.pid or pid == 0 1666 # Check the pid and loop as waitpid has been known t o
1667 # return 0 even without WNOHANG in odd situations.
1668 # http://bugs.python.org/issue14396.
1643 if pid == self.pid: 1669 if pid == self.pid:
1644 self._handle_exitstatus(sts) 1670 self._handle_exitstatus(sts)
1645 break 1671 return self.returncode
1646 finally: 1672
1647 self._waitpid_lock.release() 1673
1648 remaining = self._remaining_time(endtime) 1674 def _communicate(self, input, endtime, orig_timeout):
1649 if remaining <= 0: 1675 if self.stdin and not self._communication_started:
1650 raise TimeoutExpired(self.args, timeout) 1676 # Flush stdio buffer. This might block, if the user has
1651 delay = min(delay * 2, remaining, .05) 1677 # been writing to .stdin in an uncontrolled fashion.
1652 time.sleep(delay) 1678 self.stdin.flush()
1653 else: 1679 if not input:
1654 while self.returncode is None: 1680 self.stdin.close()
1655 with self._waitpid_lock: 1681
1656 if self.returncode is not None: 1682 stdout = None
1657 break # Another thread waited. 1683 stderr = None
1658 (pid, sts) = self._try_wait(0) 1684
1659 # Check the pid and loop as waitpid has been known to 1685 # Only create this mapping if we haven't already.
1660 # return 0 even without WNOHANG in odd situations. 1686 if not self._communication_started:
1661 # http://bugs.python.org/issue14396. 1687 self._fileobj2output = {}
1662 if pid == self.pid: 1688 if self.stdout:
1663 self._handle_exitstatus(sts) 1689 self._fileobj2output[self.stdout] = []
1664 return self.returncode 1690 if self.stderr:
1665 1691 self._fileobj2output[self.stderr] = []
1666 1692
1667 def _communicate(self, input, endtime, orig_timeout):
1668 if self.stdin and not self._communication_started:
1669 # Flush stdio buffer. This might block, if the user has
1670 # been writing to .stdin in an uncontrolled fashion.
1671 self.stdin.flush()
1672 if not input:
1673 self.stdin.close()
1674
1675 stdout = None
1676 stderr = None
1677
1678 # Only create this mapping if we haven't already.
1679 if not self._communication_started:
1680 self._fileobj2output = {}
1681 if self.stdout: 1693 if self.stdout:
1682 self._fileobj2output[self.stdout] = [] 1694 stdout = self._fileobj2output[self.stdout]
1683 if self.stderr: 1695 if self.stderr:
1684 self._fileobj2output[self.stderr] = [] 1696 stderr = self._fileobj2output[self.stderr]
1685 1697
1686 if self.stdout: 1698 self._save_input(input)
1687 stdout = self._fileobj2output[self.stdout] 1699
1688 if self.stderr: 1700 if self._input:
1689 stderr = self._fileobj2output[self.stderr] 1701 input_view = memoryview(self._input)
1690 1702
1691 self._save_input(input) 1703 with _PopenSelector() as selector:
1692 1704 if self.stdin and input:
1693 if self._input: 1705 selector.register(self.stdin, selectors.EVENT_WRITE)
1694 input_view = memoryview(self._input) 1706 if self.stdout:
1695 1707 selector.register(self.stdout, selectors.EVENT_READ)
1696 with _PopenSelector() as selector: 1708 if self.stderr:
1697 if self.stdin and input: 1709 selector.register(self.stderr, selectors.EVENT_READ)
1698 selector.register(self.stdin, selectors.EVENT_WRITE) 1710
1699 if self.stdout: 1711 while selector.get_map():
1700 selector.register(self.stdout, selectors.EVENT_READ) 1712 timeout = self._remaining_time(endtime)
1701 if self.stderr: 1713 if timeout is not None and timeout < 0:
1702 selector.register(self.stderr, selectors.EVENT_READ) 1714 raise TimeoutExpired(self.args, orig_timeout)
1703 1715
1704 while selector.get_map(): 1716 ready = selector.select(timeout)
1705 timeout = self._remaining_time(endtime) 1717 self._check_timeout(endtime, orig_timeout)
1706 if timeout is not None and timeout < 0: 1718
1707 raise TimeoutExpired(self.args, orig_timeout) 1719 # XXX Rewrite these to use non-blocking I/O on the file
1708 1720 # objects; they are no longer using C stdio!
1709 ready = selector.select(timeout) 1721
1710 self._check_timeout(endtime, orig_timeout) 1722 for key, events in ready:
1711 1723 if key.fileobj is self.stdin:
1712 # XXX Rewrite these to use non-blocking I/O on the file 1724 chunk = input_view[self._input_offset :
1713 # objects; they are no longer using C stdio! 1725 self._input_offset + _PIPE_BU F]
1714 1726 try:
1715 for key, events in ready: 1727 self._input_offset += os.write(key.fd, chunk )
1716 if key.fileobj is self.stdin: 1728 except BrokenPipeError:
1717 chunk = input_view[self._input_offset :
1718 self._input_offset + _PIPE_BUF]
1719 try:
1720 self._input_offset += os.write(key.fd, chunk)
1721 except BrokenPipeError:
1722 selector.unregister(key.fileobj)
1723 key.fileobj.close()
1724 else:
1725 if self._input_offset >= len(self._input):
1726 selector.unregister(key.fileobj) 1729 selector.unregister(key.fileobj)
1727 key.fileobj.close() 1730 key.fileobj.close()
1728 elif key.fileobj in (self.stdout, self.stderr): 1731 else:
1729 data = os.read(key.fd, 32768) 1732 if self._input_offset >= len(self._input):
1730 if not data: 1733 selector.unregister(key.fileobj)
1731 selector.unregister(key.fileobj) 1734 key.fileobj.close()
1732 key.fileobj.close() 1735 elif key.fileobj in (self.stdout, self.stderr):
1733 self._fileobj2output[key.fileobj].append(data) 1736 data = os.read(key.fd, 32768)
1734 1737 if not data:
1735 self.wait(timeout=self._remaining_time(endtime)) 1738 selector.unregister(key.fileobj)
1736 1739 key.fileobj.close()
1737 # All data exchanged. Translate lists into strings. 1740 self._fileobj2output[key.fileobj].append(data)
1738 if stdout is not None: 1741
1739 stdout = b''.join(stdout) 1742 self.wait(timeout=self._remaining_time(endtime))
1740 if stderr is not None: 1743
1741 stderr = b''.join(stderr) 1744 # All data exchanged. Translate lists into strings.
1742
1743 # Translate newlines, if requested.
1744 # This also turns bytes into strings.
1745 if self.universal_newlines:
1746 if stdout is not None: 1745 if stdout is not None:
1747 stdout = self._translate_newlines(stdout, 1746 stdout = b''.join(stdout)
1748 self.stdout.encoding)
1749 if stderr is not None: 1747 if stderr is not None:
1750 stderr = self._translate_newlines(stderr, 1748 stderr = b''.join(stderr)
1751 self.stderr.encoding) 1749
1752 1750 # Translate newlines, if requested.
1753 return (stdout, stderr) 1751 # This also turns bytes into strings.
1754 1752 if self.universal_newlines:
1755 1753 if stdout is not None:
1756 def _save_input(self, input): 1754 stdout = self._translate_newlines(stdout,
1757 # This method is called from the _communicate_with_*() methods 1755 self.stdout.encoding)
1758 # so that if we time out while communicating, we can continue 1756 if stderr is not None:
1759 # sending input if we retry. 1757 stderr = self._translate_newlines(stderr,
1760 if self.stdin and self._input is None: 1758 self.stderr.encoding)
1761 self._input_offset = 0 1759
1762 self._input = input 1760 return (stdout, stderr)
1763 if self.universal_newlines and input is not None: 1761
1764 self._input = self._input.encode(self.stdin.encoding) 1762
1765 1763 def _save_input(self, input):
1766 1764 # This method is called from the _communicate_with_*() methods
1767 def send_signal(self, sig): 1765 # so that if we time out while communicating, we can continue
1768 """Send a signal to the process.""" 1766 # sending input if we retry.
1769 # Skip signalling a process that we know has already died. 1767 if self.stdin and self._input is None:
1770 if self.returncode is None: 1768 self._input_offset = 0
1771 os.kill(self.pid, sig) 1769 self._input = input
1772 1770 if self.universal_newlines and input is not None:
1773 def terminate(self): 1771 self._input = self._input.encode(self.stdin.encoding)
1774 """Terminate the process with SIGTERM 1772
1775 """ 1773
1776 self.send_signal(signal.SIGTERM) 1774 def send_signal(self, sig):
1777 1775 """Send a signal to the process."""
1778 def kill(self): 1776 # Skip signalling a process that we know has already died.
1779 """Kill the process with SIGKILL 1777 if self.returncode is None:
1780 """ 1778 os.kill(self.pid, sig)
1781 self.send_signal(signal.SIGKILL) 1779
1780 def terminate(self):
1781 """Terminate the process with SIGTERM
1782 """
1783 self.send_signal(signal.SIGTERM)
1784
1785 def kill(self):
1786 """Kill the process with SIGKILL
1787 """
1788 self.send_signal(signal.SIGKILL)
OLDNEW
« no previous file with comments | « Lib/site.py ('k') | Lib/sysconfig.py » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+