diff --git a/Tools/README b/Tools/README index 2007e44..eae16c7 100644 --- a/Tools/README +++ b/Tools/README @@ -30,6 +30,9 @@ scripts A number of useful single-file programs, e.g. tabnanny.py tabs and spaces, and 2to3, which converts Python 2 code to Python 3 code. +sort_speed A tool for comparing the performance of list.sort for a + variety of types of data and list sizes. + ssl Currently, a tool to fetch server certificates. unicode Tools used to generate unicode database files for diff --git a/Tools/sort_speed/perf.py b/Tools/sort_speed/perf.py new file mode 100644 index 0000000..39533e5 --- /dev/null +++ b/Tools/sort_speed/perf.py @@ -0,0 +1,261 @@ +""" + +This file came from Unladen Swallow's benchmark suite, with a bunch of +stuff ripped out that isn't relevant to speed_test. It's a convenient +way to compute confidence intervals. + +The original author was jyasskin@google.com (Jeffrey Yasskin). + +""" + +import math +import re +import sys + +def avg(seq): + return sum(seq) / float(len(seq)) + + +def SampleStdDev(seq): + """Compute the standard deviation of a sample. + + Args: + seq: the numeric input data sequence. + + Returns: + The standard deviation as a float. + """ + mean = avg(seq) + squares = ((x - mean) ** 2 for x in seq) + return math.sqrt(sum(squares) / (len(seq) - 1)) + + +# A table of 95% confidence intervals for a two-tailed t distribution, as a +# function of the degrees of freedom. For larger degrees of freedom, we +# approximate. While this may look less elegant than simply calculating the +# critical value, those calculations suck. Look at +# http://www.math.unb.ca/~knight/utility/t-table.htm if you need more values. +T_DIST_95_CONF_LEVELS = [0, 12.706, 4.303, 3.182, 2.776, + 2.571, 2.447, 2.365, 2.306, 2.262, + 2.228, 2.201, 2.179, 2.160, 2.145, + 2.131, 2.120, 2.110, 2.101, 2.093, + 2.086, 2.080, 2.074, 2.069, 2.064, + 2.060, 2.056, 2.052, 2.048, 2.045, + 2.042] + + +def TDist95ConfLevel(df): + """Approximate the 95% confidence interval for Student's T distribution. + + Given the degrees of freedom, returns an approximation to the 95% + confidence interval for the Student's T distribution. + + Args: + df: An integer, the number of degrees of freedom. + + Returns: + A float. + """ + df = int(round(df)) + highest_table_df = len(T_DIST_95_CONF_LEVELS) + if df >= 200: return 1.960 + if df >= 100: return 1.984 + if df >= 80: return 1.990 + if df >= 60: return 2.000 + if df >= 50: return 2.009 + if df >= 40: return 2.021 + if df >= highest_table_df: + return T_DIST_95_CONF_LEVELS[highest_table_df - 1] + return T_DIST_95_CONF_LEVELS[df] + + +def PooledSampleVariance(sample1, sample2): + """Find the pooled sample variance for two samples. + + Args: + sample1: one sample. + sample2: the other sample. + + Returns: + Pooled sample variance, as a float. + """ + deg_freedom = len(sample1) + len(sample2) - 2 + mean1 = avg(sample1) + squares1 = ((x - mean1) ** 2 for x in sample1) + mean2 = avg(sample2) + squares2 = ((x - mean2) ** 2 for x in sample2) + + return (sum(squares1) + sum(squares2)) / float(deg_freedom) + + +def TScore(sample1, sample2): + """Calculate a t-test score for the difference between two samples. + + Args: + sample1: one sample. + sample2: the other sample. + + Returns: + The t-test score, as a float. + """ + assert len(sample1) == len(sample2) + error = PooledSampleVariance(sample1, sample2) / len(sample1) + return (avg(sample1) - avg(sample2)) / math.sqrt(error * 2) + + +def IsSignificant(sample1, sample2): + """Determine whether two samples differ significantly. + + This uses a Student's two-sample, two-tailed t-test with alpha=0.95. + + Args: + sample1: one sample. + sample2: the other sample. + + Returns: + (significant, t_score) where significant is a bool indicating whether + the two samples differ significantly; t_score is the score from the + two-sample T test. + """ + deg_freedom = len(sample1) + len(sample2) - 2 + critical_value = TDist95ConfLevel(deg_freedom) + t_score = TScore(sample1, sample2) + return (abs(t_score) >= critical_value, t_score) + + +class BenchmarkResult(object): + """An object representing data from a succesful benchmark run.""" + + def __init__(self, min_base, min_changed, delta_min, avg_base, + avg_changed, delta_avg, t_msg, std_base, std_changed, + delta_std, timeline_link): + self.min_base = min_base + self.min_changed = min_changed + self.delta_min = delta_min + self.avg_base = avg_base + self.avg_changed = avg_changed + self.delta_avg = delta_avg + self.t_msg = t_msg + self.std_base = std_base + self.std_changed = std_changed + self.delta_std = delta_std + self.timeline_link = timeline_link + + def get_timeline(self): + if self.timeline_link is None: + return "" + return "\nTimeline: %(timeline_link)s" % self.__dict__ + + def __str__(self): + return (("Avg: %(avg_base)f -> %(avg_changed)f:" + + " %(delta_avg)s\n" + self.t_msg + + "Stddev: %(std_base).5f -> %(std_changed).5f:" + + " %(delta_std)s" + self.get_timeline()) + % self.__dict__) + + +class SimpleBenchmarkResult(object): + """Object representing result data from a successful benchmark run.""" + + def __init__(self, base_time, changed_time, time_delta): + self.base_time = base_time + self.changed_time = changed_time + self.time_delta = time_delta + + def __str__(self): + return ("%(base_time)f -> %(changed_time)f: %(time_delta)s" + % self.__dict__) + + +def _FormatData(num): + return str(round(num, 2)) + +def SummarizeData(data, points=100, summary_func=max): + """Summarize a large data set using a smaller number of points. + + This will divide up the original data set into `points` windows, + using `summary_func` to summarize each window into a single point. + + Args: + data: the original data set, as a list. + points: optional; how many summary points to take. Default is 100. + summary_func: optional; function to use when summarizing each window. + Default is the max() built-in. + + Returns: + List of summary data points. + """ + window_size = int(math.ceil(len(data) / points)) + if window_size == 1: + return data + + summary_points = [] + start = 0 + while start < len(data): + end = min(start + window_size, len(data)) + summary_points.append(summary_func(data[start:end])) + start = end + return summary_points + +def TimeDelta(old, new): + if old == 0 or new == 0: + return "incomparable (one result was zero)" + if new > old: + return "%.4fx slower" % (new / old) + elif new < old: + return "%.4fx faster" % (old / new) + else: + return "no change" + + +def QuantityDelta(old, new): + if old == 0 or new == 0: + return "incomparable (one result was zero)" + if new > old: + return "%.4fx larger" % (new / old) + elif new < old: + return "%.4fx smaller" % (old / new) + else: + return "no change" + + +def CompareMultipleRuns(base_times, changed_times, options): + """Compare multiple control vs experiment runs of the same benchmark. + + Args: + base_times: iterable of float times (control). + changed_times: iterable of float times (experiment). + options: optparse.Values instance. + + Returns: + A BenchmarkResult object, summarizing the difference between the two + runs; or a SimpleBenchmarkResult object, if there was only one data + point per run. + """ + assert len(base_times) == len(changed_times) + if len(base_times) == 1: + # With only one data point, we can't do any of the interesting stats + # below. + base_time, changed_time = base_times[0], changed_times[0] + time_delta = TimeDelta(base_time, changed_time) + return SimpleBenchmarkResult(base_time, changed_time, time_delta) + + base_times = sorted(base_times) + changed_times = sorted(changed_times) + + min_base, min_changed = base_times[0], changed_times[0] + avg_base, avg_changed = avg(base_times), avg(changed_times) + std_base = SampleStdDev(base_times) + std_changed = SampleStdDev(changed_times) + delta_min = TimeDelta(min_base, min_changed) + delta_avg = TimeDelta(avg_base, avg_changed) + delta_std = QuantityDelta(std_base, std_changed) + + t_msg = "Not significant\n" + significant, t_score = IsSignificant(base_times, changed_times) + if significant: + t_msg = "Significant (t=%f)\n" % t_score + + return BenchmarkResult(min_base, min_changed, delta_min, avg_base, + avg_changed, delta_avg, t_msg, std_base, + std_changed, delta_std, None) diff --git a/Tools/sort_speed/sort_speed.py b/Tools/sort_speed/sort_speed.py new file mode 100644 index 0000000..d77e90e --- /dev/null +++ b/Tools/sort_speed/sort_speed.py @@ -0,0 +1,173 @@ +import os, sys, subprocess, re +import perf, random, time +from concurrent import futures +import argparse, collections, multiprocessing +from timeit import timeit +from math import * + +def smart_timeit(subpython, name, n, hint): + """Compute the number of times to run the command inside timeit. + + We want to use the smallest number possible that will give us + high-quality results. We first compute how long it takes to run a + simple "pass" command, which gives us a sense for our timer's + resolution. We then use determine how many times we need to run + the command inside timeit in order for the whole thing to take + 10,000 times as long as the "pass" command. + + """ + + MIN_TIME = (timeit(number=1000000) / 1e6) * 10000 + number = hint + while True: + time = basic_timeit(subpython, name, n, 1, number) + if time*number > MIN_TIME: + return number + number <<= 1 + +def basic_timeit(executable, name, n, seed, number): + output = subprocess.check_output([executable, + os.path.join(path, 'tests/support.py'), + str(n), str(seed), str(number), name]) + return float(output) + +def mytimeit(control, experiment, name, n, number, reps): + """Run an experiment in both interpreters and return the timings. + + control and experiment: python interpreter filenames + name: the test under tests/ to run + n: the size of the list + number: how many times to repeat the command within timeit + reps: how many times to repeat the experiment + + """ + ctimes = [] + etimes = [] + + # Experimentation suggests that using half of the CPUs does not + # significantly affect the timings. + pool = futures.ThreadPoolExecutor(max(1, multiprocessing.cpu_count()//2)) + jobs = [] + + for i in range(reps): + seed = random.randrange(0, 2**31-1) + jobs.append((control, name, n, seed, number)) + jobs.append((experiment, name, n, seed, number)) + + def timeit_with_args(args): + rv = basic_timeit(*args) + return rv, args + + results = pool.map(timeit_with_args, jobs) + for t, args in results: + if args[0] == control: + ctimes.append(t) + else: + etimes.append(t) + return ctimes, etimes + +def get_timing1(control, experiment, name, use_rep_map): + for i in reversed(list(range(len(ns)))): + n = ns[i] + sys.stdout.flush() + if not use_rep_map: + if i < len(ns)-1: + rep_map[n] = max(rep_map[n], rep_map[ns[i+1]]) + r = smart_timeit(control, name, n, rep_map[n]) + rep_map[n] = max(r, rep_map[n]) + r = smart_timeit(experiment, name, n, rep_map[n]) + rep_map[n] = max(r, rep_map[n]) + else: + vc, ve = mytimeit(control, experiment, name, n, rep_map[n], + args.repetitions) + summarize(name, n, vc, ve) + +def get_timing(name): + global rep_map, list_values + rep_map = {} + list_values = {} + for n in ns: + rep_map[n] = 1 + print('Calibrating timeit for', name) + get_timing1(args.control, args.experiment, name, False) + + print('Timing', name) + get_timing1(args.control, args.experiment, name, True) + +class FakeOptions: + def __init__(self, label): + self.control_label = 'control' + self.experiment_label = 'experiment' + self.benchmark_name = label + self.disable_timelines = True + +def summarize(label, n, cdata, edata): + print() + print(label, 'n =', n) + print(str(perf.CompareMultipleRuns(cdata, edata, + FakeOptions('%s-%d' % (label, n))))) + +if __name__ == '__main__': + path = os.path.dirname(__file__) + timing_d = set(name[:-3] + for name in os.listdir(os.path.join(path, 'tests/')) + if name.endswith('.py') and name != 'support.py') + + parser = argparse.ArgumentParser( + description="Compare the speed of two list implementations", + epilog="Available experiments:\n" + '\n'.join(timing_d), + formatter_class=argparse.RawDescriptionHelpFormatter) + + parser.add_argument('control', help='control/python') + parser.add_argument('experiment', help='experiment/python') + + parser.add_argument('--minn', dest='minn', type=int, default=0, + help='Minimum list size') + parser.add_argument('--maxn', dest='maxn', type=int, default=10000000, + help='Maximum list size') + parser.add_argument('-r, --repetitions', dest='repetitions', + type=int, default=11, + help='Repetitions; ' + 'how many times to repeat experiments') + parser.add_argument('tests', nargs='*', + help='Names of tests to conduct') + + args = parser.parse_args() + if os.path.isdir(args.control): + args.control = os.path.join(args.control, 'python') + if os.path.isdir(args.experiment): + args.experiment = os.path.join(args.experiment, 'python') + ns = [0] + if args.maxn > 0: + for i in range(int(round(log(args.maxn)/log(10)*10)+2)): + ns.append(int(floor(10**(i*0.1)))) + ns = [i for i in set(ns) if i >= args.minn and i <= args.maxn] + ns.sort() + + governor_filename = '/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor' + try: + with open(governor_filename) as f: + governor = f.readline().strip() + if governor != 'performance': + print('*'*72) + print() + print("Your CPU appears to have CPU scaling enabled (%s). CPU\n" + "scaling dramatically increases the variance of speed\n" + "measurements. You are strongly encouraged to set the CPU\n" + "CPU governor to 'performance' to disable CPU scaling." + % governor) + print() + print("On some systems, this might work:\n" + " sudo apt-get install cpufrequtils\n" + " sudo cpufreq-set -g performance -r\n") + print() + print('*'*72) + except IOError: + pass + + if not args.tests: + for k in sorted(timing_d): + get_timing(k) + else: + for name in args.tests: + get_timing(name) diff --git a/Tools/sort_speed/tests/sort_random.py b/Tools/sort_speed/tests/sort_random.py new file mode 100644 index 0000000..ee58844 --- /dev/null +++ b/Tools/sort_speed/tests/sort_random.py @@ -0,0 +1,9 @@ +import random +import support, sys + +support.data = list(range(support.n)) +random.shuffle(support.data) +support.t = list(support.data) +support.f = support.t.sort + +support.timelist('t[:] = data; f()', ['data', 't', 'f']) diff --git a/Tools/sort_speed/tests/sort_random_key.py b/Tools/sort_speed/tests/sort_random_key.py new file mode 100644 index 0000000..b480f44 --- /dev/null +++ b/Tools/sort_speed/tests/sort_random_key.py @@ -0,0 +1,9 @@ +import random +import support + +support.data = list(range(support.n)) +random.shuffle(support.data) +support.t = list(support.data) +support.f = support.t.sort + +support.timelist('t[:] = data; f(key=int)', ['data', 't', 'f']) diff --git a/Tools/sort_speed/tests/sort_reversed.py b/Tools/sort_speed/tests/sort_reversed.py new file mode 100644 index 0000000..5443c17 --- /dev/null +++ b/Tools/sort_speed/tests/sort_reversed.py @@ -0,0 +1,8 @@ +import random +import support + +support.data = list(range(support.n)) +support.g = support.data.reverse +support.f = support.data.sort + +support.timelist('g(); f()', ['f', 'g']) diff --git a/Tools/sort_speed/tests/sort_reversed_key.py b/Tools/sort_speed/tests/sort_reversed_key.py new file mode 100644 index 0000000..32be4a7 --- /dev/null +++ b/Tools/sort_speed/tests/sort_reversed_key.py @@ -0,0 +1,8 @@ +import random +import support + +support.data = list(range(support.n)) +support.g = support.data.reverse +support.f = support.data.sort + +support.timelist('g(); f(key=int)', ['f', 'g']) diff --git a/Tools/sort_speed/tests/sort_sorted.py b/Tools/sort_speed/tests/sort_sorted.py new file mode 100644 index 0000000..67bba05 --- /dev/null +++ b/Tools/sort_speed/tests/sort_sorted.py @@ -0,0 +1,7 @@ +import random +import support + +support.data = list(range(support.n)) +support.f = support.data.sort + +support.timelist('f()', ['f']) diff --git a/Tools/sort_speed/tests/sort_sorted_key.py b/Tools/sort_speed/tests/sort_sorted_key.py new file mode 100644 index 0000000..3ce8931 --- /dev/null +++ b/Tools/sort_speed/tests/sort_sorted_key.py @@ -0,0 +1,7 @@ +import random +import support + +support.data = list(range(support.n)) +support.f = support.data.sort + +support.timelist('f(key=int)', ['f']) diff --git a/Tools/sort_speed/tests/support.py b/Tools/sort_speed/tests/support.py new file mode 100644 index 0000000..0a19e1e --- /dev/null +++ b/Tools/sort_speed/tests/support.py @@ -0,0 +1,26 @@ +import random +import sys, os.path +from timeit import timeit +import support + +def timelist(commands, symbols): + if symbols: + setup = 'from support import %s' % ','.join(symbols) + else: + setup = '' + output = timeit(commands, setup, number=support.number) / support.number + print(output) + +def main(): + support.n = int(sys.argv[1]) + support.seed = int(sys.argv[2]) + random.seed(support.seed) + support.number = int(sys.argv[3]) + support.name = sys.argv[4] + path = os.path.dirname(__file__) + filename = os.path.join(path, '%s.py') % support.name + with open(filename) as f: + exec(compile(f.read(), filename, 'exec'), globals(), locals()) + +if __name__ == '__main__': + main()