import unittest
import subprocess
import sys
python = sys.executable
print('Executable is', python)

u_cmds = [
    " -m idlelib.CallTips",
    " -m test.test_idle", 
    " -m unittest -v idlelib.idle_test",
    " -m unittest -v idlelib.idle_test.test_calltips.Test_get_entity",
    " -m unittest -v "
        "idlelib.idle_test.test_calltips.Test_get_entity.test_bad_entity",
    ]

class Test_framework(unittest.TestCase):

    def test_cmd_(self):
        # unittest puts all output on stderr
        for cmd in u_cmds:
            cmd = python + cmd
            sub = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
            out, err = sub.communicate(timeout=10)
            self.assertIn(b"ok", err, cmd)  # verbose mode
            self.assertIn(b"Ran", err, cmd)
            self.assertIn(b"OK", err, cmd)  # verbose mode
            self.assertNotIn(b"FAIL", err, cmd)

    def test_cmd_regr(self):
        # regr puts slightly different output on stdout
        cmd = python + " -m test test_idle"
        sub = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
        out, err = sub.communicate(timeout=10)
        self.assertIn(b"test OK", out)
        self.assertNotIn(b"FAIL", out)  # not sure where FAIL would appear
        self.assertNotIn(b"FAIL", out)

if __name__ == '__main__':
    unittest.main(verbosity=2, exit=False)